From 2a8885ad937500ae217d31dd1a7d919877cae109 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:16:14 +0800 Subject: [PATCH 001/160] ADD file via upload --- array_ops_declare.cc | 130 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 array_ops_declare.cc diff --git a/array_ops_declare.cc b/array_ops_declare.cc new file mode 100644 index 00000000000..9eaa5500f80 --- /dev/null +++ b/array_ops_declare.cc @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/array_ops_declare.h" +#include + +namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 +// const +//部分语句与下面语句块作用类似，此处为对变量const进行调整 +INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 +ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* + 将value处理并存入对应ATTR_DESC结构体的相应变量中 +  将value内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key + +// Constant +//此处为对变量Constant进行调整 +INPUT_MAP(Constant) = EMPTY_INPUT_MAP; +ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; +OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  //将Constant内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为结构体 + +// ScalarSummary +INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; +ATTR_MAP(Summary) = EMPTY_ATTR_MAP; +#ifndef ENABLE_SECURITY +REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) +#endif +REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) + +// Data +INPUT_MAP(Data) = EMPTY_INPUT_MAP; +ATTR_MAP(Data) = EMPTY_ATTR_MAP; +REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) + +// Shape +INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Shape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) + +// Reshape +INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; +ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) +REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) + +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) + +// MirrorPad +INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) + +// MirrorPadGrad +INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) + +// ExpandDims +INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) + +// Squeeze +INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) + +// ReverseSequence +INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; +ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, + {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; +OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) + +// EditDistance +INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, + {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, + {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; +ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; +OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; +REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) + +// NonZero +INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; +OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) + +// Unsqueeze +INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) +} // namespace mindspore::transform -- 2.34.1 From 73f0b9142fd4e7f9fcdcb2d04045269f6f774a9f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:17:20 +0800 Subject: [PATCH 002/160] Delete 'array_ops_declare.cc' --- array_ops_declare.cc | 130 ------------------------------------------- 1 file changed, 130 deletions(-) delete mode 100644 array_ops_declare.cc diff --git a/array_ops_declare.cc b/array_ops_declare.cc deleted file mode 100644 index 9eaa5500f80..00000000000 --- a/array_ops_declare.cc +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "transform/graph_ir/op_declare/array_ops_declare.h" -#include - -namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 -// const -//部分语句与下面语句块作用类似，此处为对变量const进行调整 -INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 -ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* - 将value处理并存入对应ATTR_DESC结构体的相应变量中 -  将value内容转为字符串变量并存储至结构体的name变量中 -  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 -//对相应变量处理并存入对应AttrDesc结构体的相应变量中 -//存入MaxPool对应空间并用attr_map_指针保存 -//其中AnyTraits<>的作用为将<>内类型进行构建 -将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 -*/ -OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key - -// Constant -//此处为对变量Constant进行调整 -INPUT_MAP(Constant) = EMPTY_INPUT_MAP; -ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; -OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 -  //将Constant内容转为字符串变量并存储至结构体的name变量中 -  //引用Operator空间并将指针所指的类转为结构体 - -// ScalarSummary -INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; -ATTR_MAP(Summary) = EMPTY_ATTR_MAP; -#ifndef ENABLE_SECURITY -REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) -#endif -REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) - -// Data -INPUT_MAP(Data) = EMPTY_INPUT_MAP; -ATTR_MAP(Data) = EMPTY_ATTR_MAP; -REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) - -// Shape -INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Shape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) - -// Reshape -INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; -ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) -REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) - -// TransShape -INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; -INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; -ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) - -// MirrorPad -INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; -ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; -OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) - -// MirrorPadGrad -INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; -ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; -OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) - -// ExpandDims -INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; -ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; -OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) - -// Squeeze -INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; -OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) - -// ReverseSequence -INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; -ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, - {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; -OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) - -// EditDistance -INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, - {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, - {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; -ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; -OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; -REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) - -// NonZero -INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; -OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) - -// Unsqueeze -INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; -OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) -} // namespace mindspore::transform -- 2.34.1 From 8e88501b8c4e6d2b63055d1b8678a53766b30aa0 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:23:58 +0800 Subject: [PATCH 003/160] ADD file via upload --- mindspore2022/ccsrc/transform/graph_ir/op_declare | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_declare diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_declare b/mindspore2022/ccsrc/transform/graph_ir/op_declare new file mode 100644 index 00000000000..e69de29bb2d -- 2.34.1 From 1f97d3ae393f664e312c63c688d267fe8ad02391 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:29:02 +0800 Subject: [PATCH 004/160] ADD file via upload --- array_ops_declare.cc | 130 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 array_ops_declare.cc diff --git a/array_ops_declare.cc b/array_ops_declare.cc new file mode 100644 index 00000000000..9eaa5500f80 --- /dev/null +++ b/array_ops_declare.cc @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/array_ops_declare.h" +#include + +namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 +// const +//部分语句与下面语句块作用类似，此处为对变量const进行调整 +INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 +ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* + 将value处理并存入对应ATTR_DESC结构体的相应变量中 +  将value内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key + +// Constant +//此处为对变量Constant进行调整 +INPUT_MAP(Constant) = EMPTY_INPUT_MAP; +ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; +OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  //将Constant内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为结构体 + +// ScalarSummary +INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; +ATTR_MAP(Summary) = EMPTY_ATTR_MAP; +#ifndef ENABLE_SECURITY +REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) +#endif +REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) + +// Data +INPUT_MAP(Data) = EMPTY_INPUT_MAP; +ATTR_MAP(Data) = EMPTY_ATTR_MAP; +REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) + +// Shape +INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Shape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) + +// Reshape +INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; +ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) +REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) + +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) + +// MirrorPad +INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) + +// MirrorPadGrad +INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) + +// ExpandDims +INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) + +// Squeeze +INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) + +// ReverseSequence +INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; +ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, + {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; +OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) + +// EditDistance +INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, + {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, + {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; +ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; +OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; +REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) + +// NonZero +INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; +OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) + +// Unsqueeze +INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) +} // namespace mindspore::transform -- 2.34.1 From 0bce3c60edc109c607e2267e01adc1db55d38dca Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:29:27 +0800 Subject: [PATCH 005/160] Delete 'array_ops_declare.cc' --- array_ops_declare.cc | 130 ------------------------------------------- 1 file changed, 130 deletions(-) delete mode 100644 array_ops_declare.cc diff --git a/array_ops_declare.cc b/array_ops_declare.cc deleted file mode 100644 index 9eaa5500f80..00000000000 --- a/array_ops_declare.cc +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "transform/graph_ir/op_declare/array_ops_declare.h" -#include - -namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 -// const -//部分语句与下面语句块作用类似，此处为对变量const进行调整 -INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 -ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* - 将value处理并存入对应ATTR_DESC结构体的相应变量中 -  将value内容转为字符串变量并存储至结构体的name变量中 -  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 -//对相应变量处理并存入对应AttrDesc结构体的相应变量中 -//存入MaxPool对应空间并用attr_map_指针保存 -//其中AnyTraits<>的作用为将<>内类型进行构建 -将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 -*/ -OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key - -// Constant -//此处为对变量Constant进行调整 -INPUT_MAP(Constant) = EMPTY_INPUT_MAP; -ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; -OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 -  //将Constant内容转为字符串变量并存储至结构体的name变量中 -  //引用Operator空间并将指针所指的类转为结构体 - -// ScalarSummary -INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; -ATTR_MAP(Summary) = EMPTY_ATTR_MAP; -#ifndef ENABLE_SECURITY -REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) -#endif -REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) - -// Data -INPUT_MAP(Data) = EMPTY_INPUT_MAP; -ATTR_MAP(Data) = EMPTY_ATTR_MAP; -REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) - -// Shape -INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Shape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) - -// Reshape -INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; -ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) -REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) - -// TransShape -INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; -INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; -ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) - -// MirrorPad -INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; -ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; -OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) - -// MirrorPadGrad -INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; -ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; -OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) - -// ExpandDims -INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; -ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; -OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) - -// Squeeze -INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; -OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) - -// ReverseSequence -INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; -ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, - {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; -OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) - -// EditDistance -INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, - {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, - {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; -ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; -OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; -REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) - -// NonZero -INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; -OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) - -// Unsqueeze -INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; -OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) -} // namespace mindspore::transform -- 2.34.1 From 22ac1ddc8677b05ca7847ee342978d0b174aa84a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:29:59 +0800 Subject: [PATCH 006/160] ADD file via upload --- mindspore2022/array_ops_declare.cc | 130 +++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 mindspore2022/array_ops_declare.cc diff --git a/mindspore2022/array_ops_declare.cc b/mindspore2022/array_ops_declare.cc new file mode 100644 index 00000000000..9eaa5500f80 --- /dev/null +++ b/mindspore2022/array_ops_declare.cc @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/array_ops_declare.h" +#include + +namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 +// const +//部分语句与下面语句块作用类似，此处为对变量const进行调整 +INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 +ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* + 将value处理并存入对应ATTR_DESC结构体的相应变量中 +  将value内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key + +// Constant +//此处为对变量Constant进行调整 +INPUT_MAP(Constant) = EMPTY_INPUT_MAP; +ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; +OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  //将Constant内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为结构体 + +// ScalarSummary +INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; +ATTR_MAP(Summary) = EMPTY_ATTR_MAP; +#ifndef ENABLE_SECURITY +REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) +#endif +REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) + +// Data +INPUT_MAP(Data) = EMPTY_INPUT_MAP; +ATTR_MAP(Data) = EMPTY_ATTR_MAP; +REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) + +// Shape +INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Shape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) + +// Reshape +INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; +ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) +REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) + +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) + +// MirrorPad +INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) + +// MirrorPadGrad +INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) + +// ExpandDims +INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) + +// Squeeze +INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) + +// ReverseSequence +INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; +ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, + {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; +OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) + +// EditDistance +INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, + {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, + {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; +ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; +OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; +REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) + +// NonZero +INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; +OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) + +// Unsqueeze +INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) +} // namespace mindspore::transform -- 2.34.1 From 00e48c769409d868e6ae990dd80c2a94ee014bb4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:30:15 +0800 Subject: [PATCH 007/160] Delete 'mindspore2022/array_ops_declare.cc' --- mindspore2022/array_ops_declare.cc | 130 ----------------------------- 1 file changed, 130 deletions(-) delete mode 100644 mindspore2022/array_ops_declare.cc diff --git a/mindspore2022/array_ops_declare.cc b/mindspore2022/array_ops_declare.cc deleted file mode 100644 index 9eaa5500f80..00000000000 --- a/mindspore2022/array_ops_declare.cc +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "transform/graph_ir/op_declare/array_ops_declare.h" -#include - -namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 -// const -//部分语句与下面语句块作用类似，此处为对变量const进行调整 -INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 -ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* - 将value处理并存入对应ATTR_DESC结构体的相应变量中 -  将value内容转为字符串变量并存储至结构体的name变量中 -  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 -//对相应变量处理并存入对应AttrDesc结构体的相应变量中 -//存入MaxPool对应空间并用attr_map_指针保存 -//其中AnyTraits<>的作用为将<>内类型进行构建 -将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 -*/ -OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key - -// Constant -//此处为对变量Constant进行调整 -INPUT_MAP(Constant) = EMPTY_INPUT_MAP; -ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; -OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 -  //将Constant内容转为字符串变量并存储至结构体的name变量中 -  //引用Operator空间并将指针所指的类转为结构体 - -// ScalarSummary -INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; -ATTR_MAP(Summary) = EMPTY_ATTR_MAP; -#ifndef ENABLE_SECURITY -REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) -REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) -#endif -REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) - -// Data -INPUT_MAP(Data) = EMPTY_INPUT_MAP; -ATTR_MAP(Data) = EMPTY_ATTR_MAP; -REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) - -// Shape -INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Shape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) - -// Reshape -INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; -ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) -REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) - -// TransShape -INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; -INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; -ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; -OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) - -// MirrorPad -INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; -ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; -OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) - -// MirrorPadGrad -INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; -ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; -OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) - -// ExpandDims -INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; -ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; -OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) - -// Squeeze -INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; -OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) - -// ReverseSequence -INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; -ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, - {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; -OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) - -// EditDistance -INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, - {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, - {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; -ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; -OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; -REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) - -// NonZero -INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; -OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) - -// Unsqueeze -INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; -ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; -OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; -REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) -} // namespace mindspore::transform -- 2.34.1 From 042e46fa8672f2451b56a021875d6c7db7910a34 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:30:48 +0800 Subject: [PATCH 008/160] Delete 'mindspore2022/ccsrc/transform/graph_ir/op_declare' --- mindspore2022/ccsrc/transform/graph_ir/op_declare | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_declare diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_declare b/mindspore2022/ccsrc/transform/graph_ir/op_declare deleted file mode 100644 index e69de29bb2d..00000000000 -- 2.34.1 From ea1e6a8d0455f27729bc93c6bd65f31db50eec97 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:32:41 +0800 Subject: [PATCH 009/160] ADD file via upload --- mindspore2022/ccsrc/transform/graph_ir/op_declare | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_declare diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_declare b/mindspore2022/ccsrc/transform/graph_ir/op_declare new file mode 100644 index 00000000000..e69de29bb2d -- 2.34.1 From 758f7b0356df9dd67150ca9e3e8e6bc9280cbcd4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:33:04 +0800 Subject: [PATCH 010/160] ADD file via upload --- .../transform/graph_ir/array_ops_declare.cc | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.cc new file mode 100644 index 00000000000..9eaa5500f80 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.cc @@ -0,0 +1,130 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/array_ops_declare.h" +#include + +namespace mindspore::transform {//创建名为transform的空间，使其位于mindspore空间下 +// const +//部分语句与下面语句块作用类似，此处为对变量const进行调整 +INPUT_MAP(Const) = EMPTY_INPUT_MAP;//将const与标准进行比较，使原以input_map_为的key变为空 +ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}};/* + 将value处理并存入对应ATTR_DESC结构体的相应变量中 +  将value内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +将Const的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}};//用新建的OUTPUT结构体并与标准比较其容量，以output_map_为变量的key + +// Constant +//此处为对变量Constant进行调整 +INPUT_MAP(Constant) = EMPTY_INPUT_MAP; +ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; +OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const))//将Constant处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  //将Constant内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为结构体 + +// ScalarSummary +INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; +ATTR_MAP(Summary) = EMPTY_ATTR_MAP; +#ifndef ENABLE_SECURITY +REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) +#endif +REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) + +// Data +INPUT_MAP(Data) = EMPTY_INPUT_MAP; +ATTR_MAP(Data) = EMPTY_ATTR_MAP; +REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) + +// Shape +INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Shape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) + +// Reshape +INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; +ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) +REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) + +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) + +// MirrorPad +INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) + +// MirrorPadGrad +INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) + +// ExpandDims +INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) + +// Squeeze +INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) + +// ReverseSequence +INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; +ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, + {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; +OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) + +// EditDistance +INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, + {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, + {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; +ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; +OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; +REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) + +// NonZero +INPUT_MAP(NonZero) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(NonZero) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; +OUTPUT_MAP(NonZero) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(NonZero, kNameNonZero, ADPT_DESC(NonZero)) + +// Unsqueeze +INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; +OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) +} // namespace mindspore::transform -- 2.34.1 From 33cc371695b7816cd6be93592ac99830fbcf8291 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:33:21 +0800 Subject: [PATCH 011/160] ADD file via upload --- .../transform/graph_ir/array_ops_declare.h | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.h new file mode 100644 index 00000000000..68cf4f5e73d --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/array_ops_declare.h @@ -0,0 +1,71 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ARRAY_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ARRAY_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/array_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(Shape)//将shape收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(Shape)//将shape收录后与标准进行比较，进行空间调整，用output_map_为key储存 +//下同，根据引入的不同变量进行对不同变量的操作 + +DECLARE_OP_ADAPTER(Reshape) +DECLARE_OP_USE_OUTPUT(Reshape) + +DECLARE_OP_ADAPTER(TransShape) +DECLARE_OP_USE_INPUT_ATTR(TransShape)//将TransShape收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(TransShape) + +DECLARE_OP_ADAPTER(MirrorPad) +DECLARE_OP_USE_OUTPUT(MirrorPad) + +DECLARE_OP_ADAPTER(MirrorPadGrad) +DECLARE_OP_USE_OUTPUT(MirrorPadGrad) + +DECLARE_OP_ADAPTER(ExpandDims) +DECLARE_OP_USE_OUTPUT(ExpandDims) + +DECLARE_OP_ADAPTER(Squeeze) +DECLARE_OP_USE_OUTPUT(Squeeze) + +DECLARE_OP_ADAPTER(Constant) +DECLARE_OP_USE_OUTPUT(Constant) + +DECLARE_OP_ADAPTER(Summary) + +DECLARE_OP_ADAPTER(Const) +DECLARE_OP_USE_OUTPUT(Const) + +DECLARE_OP_ADAPTER(Data) + +DECLARE_OP_ADAPTER(ReverseSequence) +DECLARE_OP_USE_OUTPUT(ReverseSequence) + +DECLARE_OP_ADAPTER(EditDistance) +DECLARE_OP_USE_OUTPUT(EditDistance) + +DECLARE_OP_ADAPTER(NonZero) +DECLARE_OP_USE_OUTPUT(NonZero) + +DECLARE_OP_ADAPTER(Unsqueeze) +DECLARE_OP_USE_OUTPUT(Unsqueeze) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ARRAY_OPS_DECLARE_H_ -- 2.34.1 From 00683e141486d403b1eaaa4237f56266d710ec71 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:33:50 +0800 Subject: [PATCH 012/160] ADD file via upload --- .../graph_ir/control_flow_ops_declare.cc | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.cc new file mode 100644 index 00000000000..8fb94a3e50b --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.cc @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/control_flow_ops_declare.h" + +namespace mindspore::transform { +// Merge +INPUT_MAP(Merge) = EMPTY_INPUT_MAP;//将Merge与标准进行比较，使原以input_map_为的key变为空 +DYN_INPUT_MAP(Merge) = {{1, DYN_INPUT_DESC(x)}};//将Merge与标准进行比较，以dyn_input_map_为key并构造储存了name变量x的DYN_INPUT_DESC结构体并与标准比较其容量 +ATTR_MAP(Merge) = EMPTY_ATTR_MAP;//将Merge与标准进行比较，使原以input_map_为的key变为空 +OUTPUT_MAP(Merge) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(value_index)}};//将Merge与标准进行比较，并分别将两组变量与标准比较其容量 +REG_ADPT_DESC(Merge, kNameMerge, ADPT_DESC(Merge))//将Merge处理并存入对应OutputDesc结构体的相应变量中 +  //将Merge内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为OutputDesc结构体 + +// Switch +INPUT_MAP(Switch) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(pred)}}; +OUTPUT_MAP(Switch) = {{0, OUTPUT_DESC(output_false)}, {1, OUTPUT_DESC(output_true)}}; +ATTR_MAP(Switch) = EMPTY_ATTR_MAP; +REG_ADPT_DESC(Switch, kNameGeSwitch, ADPT_DESC(Switch)) +} // namespace mindspore::transform -- 2.34.1 From 3b75627087d8bb3dfa017bf9499a22dde6c24697 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:34:57 +0800 Subject: [PATCH 013/160] ADD file via upload --- .../graph_ir/control_flow_ops_declare.h | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.h new file mode 100644 index 00000000000..347ad6380b5 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/control_flow_ops_declare.h @@ -0,0 +1,46 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CONTROL_FLOW_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CONTROL_FLOW_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/control_flow_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(Merge)/* +  函数功能为将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +  具体实现过程： +    判断int与类InputDesc的容量是否小于标准+判断int与类InputDesc是否为可移动构造类型+判断int与类InputDesc是否拥有移动赋值运算符，获得IsFlat +    判断T类型是否为void，若是则返回类型Key，若不是则根据IsFlat的真假返回Key或Key const，并与T类型交换最后返回其值。 +    判断完成后，若结果为真，将范围在4~ 16384的内容从空间分配器中删去 +               若结果为假，建立新的分配器并删除原分配的空间（链表），建立新的内存块并加入到新的分配器中。 +                          统计可用元素数量，通过已分配空间计算需要分配的内存大小 +                          为新数据创建链表并加入至分配的空间中，元素字节强制对齐。 +                          并在建成的堆中分配具体空间 +    使用op_adapter_base.h下ge空间中op类下变量T，利用op_adapter.h中OpAdapter中分配的初始指针作为key储存元素 +    string与AttrDesc进行相同操作，用指针attr_map_为key存储相应内容 +*/ +//此处变量为Merge，最终用input_map_为key储存相应内容 +DECLARE_OP_USE_DYN_INPUT(Merge)//函数功能为将Merge的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(Merge)//函数功能为将Merge的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 + +DECLARE_OP_ADAPTER(Switch) +DECLARE_OP_USE_OUTPUT(Switch) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CONTROL_FLOW_OPS_DECLARE_H_ -- 2.34.1 From 0d1b846bef1dffdf58f546aea6da0c7534a5eb88 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:35:20 +0800 Subject: [PATCH 014/160] ADD file via upload --- .../transform/graph_ir/ctc_ops_declare.cc | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.cc new file mode 100644 index 00000000000..06b0a245079 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/ctc_ops_declare.h" + +namespace mindspore::transform { +// CTCLoss +INPUT_MAP(CTCLoss) = {{1, INPUT_DESC(inputs)}, + {2, INPUT_DESC(labels_indices)}, + {3, INPUT_DESC(labels_values)}, + {4, INPUT_DESC(sequence_length)}};//收录四组变量并分别与标准比较其容量并返回对应的值，以input_map_为key储存相应内容 +ATTR_MAP(CTCLoss) = { + {"preprocess_collapse_repeated", ATTR_DESC(preprocess_collapse_repeated, AnyTraits())}, + {"ctc_merge_repeated", ATTR_DESC(ctc_merge_repeated, AnyTraits())}, + {"ignore_longer_outputs_than_inputs", ATTR_DESC(ignore_longer_outputs_than_inputs, AnyTraits())}};//收录四组变量并分别与标准比较其容量并返回对应的值，以attr_map_为key储存相应内容 + /* + 将ignore_longer_outputs_than_inputs处理并存入对应ATTR_DESC结构体的相应变量中 +  将ignore_longer_outputs_than_inputs内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//AnyTraits作用:对相应变量处理并存入对应AttrDesc结构体的相应变量中 +存入MaxPool对应空间并用attr_map_指针保存 +其中AnyTraits<>的作用为将<>内类型进行构建 +*/ +OUTPUT_MAP(CTCLoss) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(gradient)}};//将收录的两组数据的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +REG_ADPT_DESC(CTCLoss, kNameCTCLoss, ADPT_DESC(CTCLoss))//将CTCLoss处理并存入对应REG_ADPT_Desc结构体的相应变量中 +  //将name变量内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为REG_ADPT_Desc结构体 + +// CTCGreedyDecoder +INPUT_MAP(CTCGreedyDecoder) = {{1, INPUT_DESC(inputs)}, {2, INPUT_DESC(sequence_length)}}; +ATTR_MAP(CTCGreedyDecoder) = {{"merge_repeated", ATTR_DESC(merge_repeated, AnyTraits())}}; +OUTPUT_MAP(CTCGreedyDecoder) = {{0, OUTPUT_DESC(decoded_indices)}, + {1, OUTPUT_DESC(decoded_values)}, + {2, OUTPUT_DESC(decoded_shape)}, + {3, OUTPUT_DESC(log_probability)}}; +REG_ADPT_DESC(CTCGreedyDecoder, kNameCTCGreedyDecoder, ADPT_DESC(CTCGreedyDecoder)) +} // namespace mindspore::transform -- 2.34.1 From dcd920656af6153ed8c0cf9fbfb878b495e3450c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:35:43 +0800 Subject: [PATCH 015/160] ADD file via upload --- .../transform/graph_ir/ctc_ops_declare.h | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.h new file mode 100644 index 00000000000..3f45b8c2070 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/ctc_ops_declare.h @@ -0,0 +1,32 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CTC_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CTC_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/ctc_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(CTCLoss)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(CTCLoss)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 + +DECLARE_OP_ADAPTER(CTCGreedyDecoder) +DECLARE_OP_USE_OUTPUT(CTCGreedyDecoder) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_CTC_OPS_DECLARE_H_ -- 2.34.1 From f4b3a069b3263eda8fb060fbb145d053538eb92d Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:36:06 +0800 Subject: [PATCH 016/160] ADD file via upload --- .../graph_ir/data_flow_ops_declare.cc | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.cc new file mode 100644 index 00000000000..b0161e3d5a2 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.cc @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/data_flow_ops_declare.h" +#include + +namespace mindspore::transform { +INPUT_MAP(TensorArray) = {{1, INPUT_DESC(size)}};/* + 将value处理并存入对应ATTR_DESC结构体的相应变量中 +  将value内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +将TensorArray的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +ATTR_MAP(TensorArray) = {{"dtype", ATTR_DESC(dtype, AnyTraits())},/* + 将dtype处理并存入对应ATTR_DESC结构体的相应变量中 +  将dtype内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建*/ + {"element_shape", ATTR_DESC(element_shape, AnyTraits>())}, + {"dynamic_size", ATTR_DESC(dynamic_size, AnyTraits())}, + {"clear_after_read", ATTR_DESC(clear_after_read, AnyTraits())}, + {"identical_element_shapes", ATTR_DESC(identical_element_shapes, AnyTraits())}, + {"tensor_array_name", ATTR_DESC(tensor_array_name, AnyTraits())}};//将TensorArray的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +OUTPUT_MAP(TensorArray) = {{0, OUTPUT_DESC(handle)}, {1, OUTPUT_DESC(flow)}};/* + 将handle和flow处理并存入对应ATTR_DESC结构体的相应变量中 +  将变量内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体*/ +//将TensorArray的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +REG_ADPT_DESC(TensorArray, kNameTensorArray, ADPT_DESC(TensorArray))/* + 将TensorArray处理并存入对应ADPT_DESC结构体的相应变量中 +  将TensorArray内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将TensorArray处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将TensorArray内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +INPUT_MAP(TensorArrayWrite) = { + {1, INPUT_DESC(handle)}, {2, INPUT_DESC(index)}, {3, INPUT_DESC(value)}, {4, INPUT_DESC(flow_in)}}; +ATTR_MAP(TensorArrayWrite) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TensorArrayWrite) = {{0, OUTPUT_DESC(flow_out)}}; +REG_ADPT_DESC(TensorArrayWrite, kNameTensorArrayWrite, ADPT_DESC(TensorArrayWrite)) + +INPUT_MAP(TensorArrayGather) = {{1, INPUT_DESC(handle)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(flow_in)}}; +ATTR_MAP(TensorArrayGather) = {{"dtype", ATTR_DESC(dtype, AnyTraits())}, + {"element_shape", ATTR_DESC(element_shape, AnyTraits>())}}; +OUTPUT_MAP(TensorArrayGather) = {{0, OUTPUT_DESC(value)}}; +REG_ADPT_DESC(TensorArrayGather, kNameTensorArrayGather, ADPT_DESC(TensorArrayGather)) +} // namespace mindspore::transform -- 2.34.1 From e35f71777675b6c69c9539454e7dce5bbfc75a8c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:36:29 +0800 Subject: [PATCH 017/160] ADD file via upload --- .../graph_ir/data_flow_ops_declare.h | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.h new file mode 100644 index 00000000000..51455e15e69 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/data_flow_ops_declare.h @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_DATA_FLOW_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_DATA_FLOW_OPS_DECLARE_H_ + +#include +#include +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/data_flow_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(TensorArray)//将TensorArray收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(TensorArray)//将TensorArray收录后与标准进行比较，进行空间调整，用output_map_为key储存 + +DECLARE_OP_ADAPTER(TensorArrayWrite) +DECLARE_OP_USE_OUTPUT(TensorArrayWrite) + +DECLARE_OP_ADAPTER(TensorArrayGather) +DECLARE_OP_USE_OUTPUT(TensorArrayGather) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_DATA_FLOW_OPS_DECLARE_H_ -- 2.34.1 From a75dd337d5ff1e1d67aa571583df4d3c128ccc7e Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:37:04 +0800 Subject: [PATCH 018/160] ADD file via upload --- .../elewise_calculation_ops_declare.cc | 674 ++++++++++++++++++ 1 file changed, 674 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.cc new file mode 100644 index 00000000000..7920ee91fde --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.cc @@ -0,0 +1,674 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/elewise_calculation_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// Assign +INPUT_MAP(Assign) = {{1, INPUT_DESC(ref)}, {2, INPUT_DESC(value)}};/* + 将value和ref处理并存入对应INPUT_DESC结构体的相应变量中 +  将value和ref内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将Assign的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +ATTR_MAP(Assign) = EMPTY_ATTR_MAP;//将Assign与标准进行比较，使原以attr_map_为储存信息的key变为空 +OUTPUT_MAP(Assign) = {{0, OUTPUT_DESC(ref)}};/* + 将ref处理并存入对应OUTPUT_DESC结构体的相应变量中 +  将ref内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体 +将Assign的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ +REG_ADPT_DESC(Assign, prim::kPrimAssign->name(), ADPT_DESC(Assign))/* + 将Assign处理并存入对应ADPT_DESC结构体的相应变量中 +  将Assign内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将Assign处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将Assign内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +REG_ADPT_DESC(StateSetItem, prim::kPrimStateSetItem->name(), ADPT_DESC(Assign)) + +// add +INPUT_MAP(Add) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Add) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Add) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Add, prim::kPrimAdd->name(), + std::make_shared( + std::make_shared>(ExtraAttr({{"mode", MakeValue(static_cast(1))}})), + std::make_shared>(ExtraAttr({{"mode", MakeValue(static_cast(1))}})))) + +// AccumulateNV2 +INPUT_MAP(AccumulateNV2) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(AccumulateNV2) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(AccumulateNV2) = {{"n", ATTR_DESC(N, AnyTraits())}}; +OUTPUT_MAP(AccumulateNV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(AccumulateNV2, kNameAccumulateNV2, ADPT_DESC(AccumulateNV2)) + +// ConfusionMulGrad +INPUT_MAP(ConfusionMulGrad) = {{1, INPUT_DESC(input0)}, {2, INPUT_DESC(input1)}, {3, INPUT_DESC(input2)}}; +ATTR_MAP(ConfusionMulGrad) = {{"axes", ATTR_DESC(axes, AnyTraits>())}, + {"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ConfusionMulGrad) = {{0, OUTPUT_DESC(output0)}, {1, OUTPUT_DESC(output1)}}; +REG_ADPT_DESC(ConfusionMulGrad, kNameConfusionMulGrad, ADPT_DESC(ConfusionMulGrad)) + +// FakeQuantWithMinMaxVars +INPUT_MAP(FakeQuantWithMinMaxVars) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(min)}, {3, INPUT_DESC(max)}}; +ATTR_MAP(FakeQuantWithMinMaxVars) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits())}, + {"narrow_range", ATTR_DESC(narrow_range, AnyTraits())}}; +OUTPUT_MAP(FakeQuantWithMinMaxVars) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(FakeQuantWithMinMaxVars, kNameFakeQuantWithMinMaxVars, ADPT_DESC(FakeQuantWithMinMaxVars)) + +// FakeQuantWithMinMaxVarsGradient +INPUT_MAP(FakeQuantWithMinMaxVarsGradient) = { + {1, INPUT_DESC(gradients)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(min)}, {4, INPUT_DESC(max)}}; +ATTR_MAP(FakeQuantWithMinMaxVarsGradient) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits())}, + {"narrow_range", ATTR_DESC(narrow_range, AnyTraits())}}; +OUTPUT_MAP(FakeQuantWithMinMaxVarsGradient) = { + {0, OUTPUT_DESC(backprops_wrt_x)}, {1, OUTPUT_DESC(backprops_wrt_min)}, {2, OUTPUT_DESC(backprops_wrt_max)}}; +REG_ADPT_DESC(FakeQuantWithMinMaxVarsGradient, kNameFakeQuantWithMinMaxVarsGradient, + ADPT_DESC(FakeQuantWithMinMaxVarsGradient)) + +// FakeQuantWithMinMaxVarsPerChannel +INPUT_MAP(FakeQuantWithMinMaxVarsPerChannel) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(min)}, {3, INPUT_DESC(max)}}; +ATTR_MAP(FakeQuantWithMinMaxVarsPerChannel) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits())}, + {"narrow_range", ATTR_DESC(narrow_range, AnyTraits())}}; +OUTPUT_MAP(FakeQuantWithMinMaxVarsPerChannel) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(FakeQuantWithMinMaxVarsPerChannel, kNameFakeQuantWithMinMaxVarsPerChannel, + ADPT_DESC(FakeQuantWithMinMaxVarsPerChannel)) + +// FakeQuantWithMinMaxVarsPerChannelGradient +INPUT_MAP(FakeQuantWithMinMaxVarsPerChannelGradient) = { + {1, INPUT_DESC(gradients)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(min)}, {4, INPUT_DESC(max)}}; +ATTR_MAP(FakeQuantWithMinMaxVarsPerChannelGradient) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits())}, + {"narrow_range", ATTR_DESC(narrow_range, AnyTraits())}}; +OUTPUT_MAP(FakeQuantWithMinMaxVarsPerChannelGradient) = { + {0, OUTPUT_DESC(backprops_wrt_x)}, {1, OUTPUT_DESC(backprops_wrt_min)}, {2, OUTPUT_DESC(backprops_wrt_max)}}; +REG_ADPT_DESC(FakeQuantWithMinMaxVarsPerChannelGradient, kNameFakeQuantWithMinMaxVarsPerChannelGradient, + ADPT_DESC(FakeQuantWithMinMaxVarsPerChannelGradient)) + +// GreaterEqual +INPUT_MAP(GreaterEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(GreaterEqual) = EMPTY_ATTR_MAP; +OUTPUT_MAP(GreaterEqual) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(GreaterEqual, kNameGreaterEqual, ADPT_DESC(GreaterEqual)) + +// AssignAdd +INPUT_MAP(AssignAdd) = {{1, INPUT_DESC(ref)}, {2, INPUT_DESC(value)}}; +ATTR_MAP(AssignAdd) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AssignAdd) = {{0, OUTPUT_DESC(ref)}}; +REG_ADPT_DESC(AssignAdd, kNameAssignAdd, ADPT_DESC(AssignAdd)) + +// AssignSub +INPUT_MAP(AssignSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(value)}}; +ATTR_MAP(AssignSub) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AssignSub) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(AssignSub, kNameAssignSub, ADPT_DESC(AssignSub)) + +// Cos +INPUT_MAP(Cos) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Cos) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Cos) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Cos, kNameCos, ADPT_DESC(Cos)) + +// Cosh +INPUT_MAP(Cosh) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Cosh) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Cosh) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Cosh, kNameCosh, ADPT_DESC(Cosh)) + +// Acos +INPUT_MAP(Acos) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Acos) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Acos) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Acos, kNameACos, ADPT_DESC(Acos)) + +// AcosGrad +INPUT_MAP(AcosGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AcosGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AcosGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(AcosGrad, kNameACosGrad, ADPT_DESC(AcosGrad)) + +// Acosh +INPUT_MAP(Acosh) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Acosh) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Acosh) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Acosh, kNameAcosh, ADPT_DESC(Acosh)) + +// AcoshGrad +INPUT_MAP(AcoshGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AcoshGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AcoshGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(AcoshGrad, kNameAcoshGrad, ADPT_DESC(AcoshGrad)) + +// Div +INPUT_MAP(Div) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Div) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Div) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Div, kNameDiv, ADPT_DESC(Div)) + +// TruncateDiv +INPUT_MAP(TruncateDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(TruncateDiv) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TruncateDiv) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TruncateDiv, kNameTruncateDiv, ADPT_DESC(TruncateDiv)) + +// TruncateMod +INPUT_MAP(TruncateMod) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(TruncateMod) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TruncateMod) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TruncateMod, kNameTruncateMod, ADPT_DESC(TruncateMod)) + +// Xlogy +INPUT_MAP(Xlogy) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Xlogy) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Xlogy) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Xlogy, kNameXlogy, ADPT_DESC(Xlogy)) + +// DivNoNan +INPUT_MAP(DivNoNan) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(DivNoNan) = EMPTY_ATTR_MAP; +OUTPUT_MAP(DivNoNan) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(DivNoNan, kNameDivNoNan, ADPT_DESC(DivNoNan)) + +// Floor +INPUT_MAP(Floor) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Floor) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Floor) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Floor, kNameFloor, ADPT_DESC(Floor)) + +// FloorDiv +INPUT_MAP(FloorDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(FloorDiv) = EMPTY_ATTR_MAP; +OUTPUT_MAP(FloorDiv) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(FloorDiv, kNameFloorDiv, ADPT_DESC(FloorDiv)) + +// FloorMod +INPUT_MAP(FloorMod) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(FloorMod) = EMPTY_ATTR_MAP; +OUTPUT_MAP(FloorMod) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(FloorMod, kNameFloorMod, ADPT_DESC(FloorMod)) + +// Sin +INPUT_MAP(Sin) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sin) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sin) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Sin, kNameSin, ADPT_DESC(Sin)) + +// Sinh +INPUT_MAP(Sinh) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sinh) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sinh) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Sinh, kNameSinh, ADPT_DESC(Sinh)) + +// Asin +INPUT_MAP(Asin) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Asin) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Asin) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Asin, kNameAsin, ADPT_DESC(Asin)) + +// AsinGrad +INPUT_MAP(AsinGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AsinGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AsinGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(AsinGrad, kNameAsinGrad, ADPT_DESC(AsinGrad)) + +// Asinh +INPUT_MAP(Asinh) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Asinh) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Asinh) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Asinh, kNameAsinh, ADPT_DESC(Asinh)) + +// AsinhGrad +INPUT_MAP(AsinhGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AsinhGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AsinhGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(AsinhGrad, kNameAsinhGrad, ADPT_DESC(AsinhGrad)) + +// BitwiseAnd +INPUT_MAP(BitwiseAnd) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BitwiseAnd) = EMPTY_ATTR_MAP; +OUTPUT_MAP(BitwiseAnd) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BitwiseAnd, kNameBitwiseAnd, ADPT_DESC(BitwiseAnd)) + +// BitwiseOr +INPUT_MAP(BitwiseOr) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BitwiseOr) = EMPTY_ATTR_MAP; +OUTPUT_MAP(BitwiseOr) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BitwiseOr, kNameBitwiseOr, ADPT_DESC(BitwiseOr)) + +// BitwiseXor +INPUT_MAP(BitwiseXor) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BitwiseXor) = EMPTY_ATTR_MAP; +OUTPUT_MAP(BitwiseXor) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BitwiseXor, kNameBitwiseXor, ADPT_DESC(BitwiseXor)) + +// Ceil +INPUT_MAP(Ceil) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Ceil) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Ceil) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Ceil, kNameCeil, ADPT_DESC(Ceil)) + +// CosineEmbeddingLoss +INPUT_MAP(CosineEmbeddingLoss) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(target)}}; +ATTR_MAP(CosineEmbeddingLoss) = {{"margin", ATTR_DESC(margin, AnyTraits())}, + {"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(CosineEmbeddingLoss) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(CosineEmbeddingLoss, kNameCosineEmbeddingLoss, ADPT_DESC(CosineEmbeddingLoss)) + +// Xdivy +INPUT_MAP(Xdivy) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Xdivy) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Xdivy) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Xdivy, kNameXdivy, ADPT_DESC(Xdivy)) + +// Mod +INPUT_MAP(Mod) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Mod) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Mod) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Mod, kNameMod, ADPT_DESC(Mod)) + +// Exp +INPUT_MAP(Exp) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Exp) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Exp) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Exp, kNameExp, ADPT_DESC(Exp)) + +// Expm1 +INPUT_MAP(Expm1) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Expm1) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Expm1) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Expm1, kNameExpm1, ADPT_DESC(Expm1)) + +// BiasAdd +INPUT_MAP(BiasAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(bias)}}; +ATTR_MAP(BiasAdd) = {{"format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(BiasAdd) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BiasAdd, kNameBiasAdd, ADPT_DESC(BiasAdd)) + +// ZerosLike +INPUT_MAP(ZerosLike) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ZerosLike) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ZerosLike) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ZerosLike, kNameZerosLike, ADPT_DESC(ZerosLike)) + +// OnesLike +INPUT_MAP(OnesLike) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(OnesLike) = EMPTY_ATTR_MAP; +OUTPUT_MAP(OnesLike) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(OnesLike, kNameOnesLike, ADPT_DESC(OnesLike)) + +// ArgMaxD +INPUT_MAP(ArgMaxD) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMaxD) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"output_type", ATTR_DESC(dtype, AnyTraits())}}; +OUTPUT_MAP(ArgMaxD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ArgMaxD, kNameArgmax, ADPT_DESC(ArgMaxD)) + +// ArgMaxV2 +INPUT_MAP(ArgMaxV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(dimension)}}; +ATTR_MAP(ArgMaxV2) = {{"output_type", ATTR_DESC(dtype, AnyTraits())}}; +OUTPUT_MAP(ArgMaxV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ArgMaxV2, kNameArgMaxV2, ADPT_DESC(ArgMaxV2)) + +// ArgMinD +INPUT_MAP(ArgMinD) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMinD) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"output_type", ATTR_DESC(dtype, AnyTraits())}}; +OUTPUT_MAP(ArgMinD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ArgMinD, kNameArgmin, ADPT_DESC(ArgMinD)) + +// ArgMaxWithValue +INPUT_MAP(ArgMaxWithValue) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMaxWithValue) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ArgMaxWithValue) = {{0, OUTPUT_DESC(indice)}, {1, OUTPUT_DESC(values)}}; +REG_ADPT_DESC(ArgMaxWithValue, kNameArgMaxWithValue, ADPT_DESC(ArgMaxWithValue)) + +// ArgMinWithValue +INPUT_MAP(ArgMinWithValue) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ArgMinWithValue) = {{"axis", ATTR_DESC(dimension, AnyTraits())}, + {"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +OUTPUT_MAP(ArgMinWithValue) = {{0, OUTPUT_DESC(indice)}, {1, OUTPUT_DESC(values)}}; +REG_ADPT_DESC(ArgMinWithValue, kNameArgMinWithValue, ADPT_DESC(ArgMinWithValue)) + +// Rint +INPUT_MAP(Rint) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Rint) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Rint) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Rint, kNameRint, ADPT_DESC(Rint)) + +// BesselI0e +INPUT_MAP(BesselI0e) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(BesselI0e) = EMPTY_ATTR_MAP; +OUTPUT_MAP(BesselI0e) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BesselI0e, kNameBesselI0e, ADPT_DESC(BesselI0e)) + +// BesselI1e +INPUT_MAP(BesselI1e) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(BesselI1e) = EMPTY_ATTR_MAP; +OUTPUT_MAP(BesselI1e) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BesselI1e, kNameBesselI1e, ADPT_DESC(BesselI1e)) + +// Inv +INPUT_MAP(Inv) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Inv) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Inv) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Inv, kNameInv, ADPT_DESC(Inv)) + +// InvGrad +INPUT_MAP(InvGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}}; +ATTR_MAP(InvGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(InvGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(InvGrad, kNameInvGrad, ADPT_DESC(InvGrad)) + +// Invert +INPUT_MAP(Invert) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Invert) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Invert) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Invert, kNameInvert, ADPT_DESC(Invert)) + +// Log1p +INPUT_MAP(Log1p) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Log1p) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Log1p) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Log1p, kNameLog1p, ADPT_DESC(Log1p)) + +// RsqrtGrad +INPUT_MAP(RsqrtGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(RsqrtGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(RsqrtGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(RsqrtGrad, kNameRsqrtGrad, ADPT_DESC(RsqrtGrad)) + +// SqrtGrad +INPUT_MAP(SqrtGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(SqrtGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SqrtGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(SqrtGrad, kNameSqrtGrad, ADPT_DESC(SqrtGrad)) + +// ReciprocalGrad +INPUT_MAP(ReciprocalGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(ReciprocalGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ReciprocalGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(ReciprocalGrad, kNameReciprocalGrad, ADPT_DESC(ReciprocalGrad)) + +// AddN +INPUT_MAP(AddN) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(AddN) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(AddN) = {{"n", ATTR_DESC(N, AnyTraits())}}; +OUTPUT_MAP(AddN) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(AddN, kNameAddN, ADPT_DESC(AddN)) + +// Mul +INPUT_MAP(Mul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Mul) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Mul) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Mul, prim::kPrimMul->name(), ADPT_DESC(Mul)) + +// MulNoNan +INPUT_MAP(MulNoNan) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(MulNoNan) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MulNoNan) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MulNoNan, kNameMulNoNan, ADPT_DESC(MulNoNan)) + +// RealDiv +INPUT_MAP(RealDiv) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(RealDiv) = EMPTY_ATTR_MAP; +OUTPUT_MAP(RealDiv) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(RealDiv, kNameRealDiv, ADPT_DESC(RealDiv)) + +// Cast +INPUT_MAP(Cast) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(Cast) = {{2, ATTR_DESC(dst_type, AnyTraits())}}; +ATTR_MAP(Cast) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Cast) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Cast, prim::kPrimCast->name(), ADPT_DESC(Cast)) + +// Reciprocal +INPUT_MAP(Reciprocal) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Reciprocal) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Reciprocal) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Reciprocal, kNameReciprocal, ADPT_DESC(Reciprocal)) + +// Sub +INPUT_MAP(Sub) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Sub) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sub) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Sub, prim::kPrimSub->name(), ADPT_DESC(Sub)) + +// Neg +INPUT_MAP(Neg) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Neg) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Neg) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Neg, prim::kPrimNeg->name(), ADPT_DESC(Neg)) + +// Less +INPUT_MAP(Less) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Less) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Less) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Less, kNameLess, ADPT_DESC(Less)) + +// Rsqrt +INPUT_MAP(Rsqrt) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Rsqrt) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Rsqrt) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Rsqrt, kNameRsqrt, ADPT_DESC(Rsqrt)) + +// Sqrt +INPUT_MAP(Sqrt) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sqrt) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sqrt) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Sqrt, kNameSqrt, ADPT_DESC(Sqrt)) + +// Square +INPUT_MAP(Square) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Square) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Square) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Square, kNameSquare, ADPT_DESC(Square)) + +// SquaredDifference +INPUT_MAP(SquaredDifference) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(SquaredDifference) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SquaredDifference) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(SquaredDifference, kNameSquaredDifference, ADPT_DESC(SquaredDifference)) + +// SquareSumAll +INPUT_MAP(SquareSumAll) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(SquareSumAll) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SquareSumAll) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; +REG_ADPT_DESC(SquareSumAll, kNameSquareSumAll, ADPT_DESC(SquareSumAll)) + +// Maximum +INPUT_MAP(Maximum) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Maximum) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Maximum) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Maximum, prim::kPrimMaximum->name(), ADPT_DESC(Maximum)) + +// Minimum +INPUT_MAP(Minimum) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Minimum) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Minimum) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Minimum, prim::kPrimMinimum->name(), ADPT_DESC(Minimum)) + +// MaximumGrad +INPUT_MAP(MaximumGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grads)}}; +ATTR_MAP(MaximumGrad) = {{"grad_x", ATTR_DESC(grad_x, AnyTraits())}, + {"grad_y", ATTR_DESC(grad_y, AnyTraits())}}; +OUTPUT_MAP(MaximumGrad) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; +REG_ADPT_DESC(MaximumGrad, prim::kPrimMaximumGrad->name(), ADPT_DESC(MaximumGrad)) + +// MinimumGrad +INPUT_MAP(MinimumGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grads)}}; +ATTR_MAP(MinimumGrad) = {{"grad_x", ATTR_DESC(grad_x, AnyTraits())}, + {"grad_y", ATTR_DESC(grad_y, AnyTraits())}}; +OUTPUT_MAP(MinimumGrad) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; +REG_ADPT_DESC(MinimumGrad, prim::kPrimMinimumGrad->name(), ADPT_DESC(MinimumGrad)) + +// Pow +INPUT_MAP(Pow) = { + {1, INPUT_DESC(x1)}, + {2, INPUT_DESC(x2)}, +}; +ATTR_MAP(Pow) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Pow) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Pow, kNamePow, ADPT_DESC(Pow)) + +// PopulationCount +INPUT_MAP(PopulationCount) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(PopulationCount) = EMPTY_ATTR_MAP; +OUTPUT_MAP(PopulationCount) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(PopulationCount, kNamePopulationCount, ADPT_DESC(PopulationCount)) + +// Equal +INPUT_MAP(Equal) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Equal) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Equal) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Equal, kNameEqual, ADPT_DESC(Equal)) + +// ApproximateEqual +INPUT_MAP(ApproximateEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(ApproximateEqual) = {{"tolerance", ATTR_DESC(tolerance, AnyTraits())}}; +OUTPUT_MAP(ApproximateEqual) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ApproximateEqual, kNameApproximateEqual, ADPT_DESC(ApproximateEqual)) + +// NotEqual +INPUT_MAP(NotEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(NotEqual) = EMPTY_ATTR_MAP; +OUTPUT_MAP(NotEqual) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(NotEqual, kNameNotEqual, ADPT_DESC(NotEqual)) + +// Log +INPUT_MAP(Log) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Log) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Log) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Log, kNameLog, ADPT_DESC(Log)) + +// LogicalAnd +INPUT_MAP(LogicalAnd) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(LogicalAnd) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LogicalAnd) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(LogicalAnd, kNameLogicalAnd, ADPT_DESC(LogicalAnd)) + +// LogicalOr +INPUT_MAP(LogicalOr) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(LogicalOr) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LogicalOr) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(LogicalOr, kNameLogicalOr, ADPT_DESC(LogicalOr)) + +// LogicalNot +INPUT_MAP(LogicalNot) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(LogicalNot) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LogicalNot) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(LogicalNot, kNameLogicalNot, ADPT_DESC(LogicalNot)) + +// Greater +INPUT_MAP(Greater) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Greater) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Greater) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Greater, kNameGreater, ADPT_DESC(Greater)) + +// LessEqual +INPUT_MAP(LessEqual) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(LessEqual) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LessEqual) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(LessEqual, kNameLessEqual, ADPT_DESC(LessEqual)) + +// Abs +INPUT_MAP(Abs) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Abs) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Abs) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Abs, kNameAbs, ADPT_DESC(Abs)) + +// AbsGrad +INPUT_MAP(AbsGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AbsGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AbsGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(AbsGrad, kNameAbsGrad, ADPT_DESC(AbsGrad)) + +// Sign +INPUT_MAP(Sign) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sign) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Sign) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Sign, kNameSign, ADPT_DESC(Sign)) + +// Round +INPUT_MAP(Round) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Round) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Round) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Round, kNameRound, ADPT_DESC(Round)) + +// Tan +INPUT_MAP(Tan) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Tan) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Tan) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Tan, kNameTan, ADPT_DESC(Tan)) + +// Atan +INPUT_MAP(Atan) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Atan) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Atan) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Atan, kNameAtan, ADPT_DESC(Atan)) + +// AtanGrad +INPUT_MAP(AtanGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +ATTR_MAP(AtanGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(AtanGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(AtanGrad, kNameAtanGrad, ADPT_DESC(AtanGrad)) + +// Atanh +INPUT_MAP(Atanh) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Atanh) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Atanh) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Atanh, kNameAtanh, ADPT_DESC(Atanh)) + +// Atan2 +INPUT_MAP(Atan2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(Atan2) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Atan2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Atan2, kNameAtan2, ADPT_DESC(Atan2)) + +// LambApplyOptimizerAssign +INPUT_MAP(LambApplyOptimizerAssign) = { + {1, INPUT_DESC(grad)}, {2, INPUT_DESC(inputv)}, {3, INPUT_DESC(inputm)}, + {4, INPUT_DESC(input3)}, {5, INPUT_DESC(mul0_x)}, {6, INPUT_DESC(mul1_x)}, + {7, INPUT_DESC(mul2_x)}, {8, INPUT_DESC(mul3_x)}, {9, INPUT_DESC(add2_y)}, + {10, INPUT_DESC(steps)}, {11, INPUT_DESC(do_use_weight)}, {12, INPUT_DESC(weight_decay_rate)}}; +ATTR_MAP(LambApplyOptimizerAssign) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LambApplyOptimizerAssign) = {{0, OUTPUT_DESC(output0)}, {1, OUTPUT_DESC(inputv)}, {2, OUTPUT_DESC(inputm)}}; +REG_ADPT_DESC(LambApplyOptimizerAssign, kNameLambApplyOptimizerAssign, ADPT_DESC(LambApplyOptimizerAssign)) + +// LambApplyWeightAssign +INPUT_MAP(LambApplyWeightAssign) = {{1, INPUT_DESC(input0)}, + {2, INPUT_DESC(input1)}, + {3, INPUT_DESC(input2)}, + {4, INPUT_DESC(input3)}, + {5, INPUT_DESC(input_param)}}; +ATTR_MAP(LambApplyWeightAssign) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LambApplyWeightAssign) = {{0, OUTPUT_DESC(input_param)}}; +REG_ADPT_DESC(LambApplyWeightAssign, kNameLambApplyWeightAssign, ADPT_DESC(LambApplyWeightAssign)) + +// Eltwise +INPUT_MAP(Eltwise) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(Eltwise) = {{1, DYN_INPUT_DESC(x)}}; +ATTR_MAP(Eltwise) = {{"n", ATTR_DESC(N, AnyTraits())}, + {"mode", ATTR_DESC(mode, AnyTraits())}, + {"coeff", ATTR_DESC(coeff, AnyTraits>(), AnyTraits())}}; +OUTPUT_MAP(Eltwise) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Eltwise, kNameEltwise, ADPT_DESC(Eltwise)) +} // namespace mindspore::transform -- 2.34.1 From cace16496483b12f17251be727feb082c10c9965 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:37:27 +0800 Subject: [PATCH 019/160] ADD file via upload --- .../elewise_calculation_ops_declare.h | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.h new file mode 100644 index 00000000000..d7535bdbf41 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/elewise_calculation_ops_declare.h @@ -0,0 +1,326 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ELEWISE_CALCULATION_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ELEWISE_CALCULATION_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/elewise_calculation_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(AccumulateNV2)//将AccumulateNV2的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_DYN_INPUT(AccumulateNV2)//将AccumulateNV2的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(AccumulateNV2)//将AccumulateNV2的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 + +DECLARE_OP_ADAPTER(ConfusionMulGrad) +DECLARE_OP_USE_OUTPUT(ConfusionMulGrad) + +DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVars) +DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVars) + +DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVarsGradient) +DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVarsGradient) + +DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVarsPerChannel) +DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVarsPerChannel) + +DECLARE_OP_ADAPTER(FakeQuantWithMinMaxVarsPerChannelGradient) +DECLARE_OP_USE_OUTPUT(FakeQuantWithMinMaxVarsPerChannelGradient) + +DECLARE_OP_ADAPTER(GreaterEqual) +DECLARE_OP_USE_OUTPUT(GreaterEqual) + +DECLARE_OP_ADAPTER(AssignAdd) +DECLARE_OP_USE_OUTPUT(AssignAdd) + +DECLARE_OP_ADAPTER(AssignSub) +DECLARE_OP_USE_OUTPUT(AssignSub) + +DECLARE_OP_ADAPTER(ZerosLike) +DECLARE_OP_USE_OUTPUT(ZerosLike) + +DECLARE_OP_ADAPTER(OnesLike) +DECLARE_OP_USE_OUTPUT(OnesLike) + +DECLARE_OP_ADAPTER(ArgMaxD) +DECLARE_OP_USE_OUTPUT(ArgMaxD) + +DECLARE_OP_ADAPTER(ArgMaxV2) +DECLARE_OP_USE_OUTPUT(ArgMaxV2) + +DECLARE_OP_ADAPTER(ArgMinD) +DECLARE_OP_USE_OUTPUT(ArgMinD) + +DECLARE_OP_ADAPTER(ArgMaxWithValue) +DECLARE_OP_USE_OUTPUT(ArgMaxWithValue) + +DECLARE_OP_ADAPTER(ArgMinWithValue) +DECLARE_OP_USE_OUTPUT(ArgMinWithValue) + +DECLARE_OP_ADAPTER(Mul) +DECLARE_OP_USE_OUTPUT(Mul) + +DECLARE_OP_ADAPTER(MulNoNan) +DECLARE_OP_USE_OUTPUT(MulNoNan) + +DECLARE_OP_ADAPTER(AddN) +DECLARE_OP_USE_DYN_INPUT(AddN) +DECLARE_OP_USE_OUTPUT(AddN) + +DECLARE_OP_ADAPTER(Less) +DECLARE_OP_USE_OUTPUT(Less) + +DECLARE_OP_ADAPTER(Rsqrt) +DECLARE_OP_USE_OUTPUT(Rsqrt) + +DECLARE_OP_ADAPTER(Sqrt) +DECLARE_OP_USE_OUTPUT(Sqrt) + +DECLARE_OP_ADAPTER(Square) +DECLARE_OP_USE_OUTPUT(Square) + +DECLARE_OP_ADAPTER(SquaredDifference) +DECLARE_OP_USE_OUTPUT(SquaredDifference) + +DECLARE_OP_ADAPTER(SquareSumAll) +DECLARE_OP_USE_OUTPUT(SquareSumAll) + +DECLARE_OP_ADAPTER(Maximum) +DECLARE_OP_USE_OUTPUT(Maximum) + +DECLARE_OP_ADAPTER(Minimum) +DECLARE_OP_USE_OUTPUT(Minimum) + +DECLARE_OP_ADAPTER(MaximumGrad) +DECLARE_OP_USE_OUTPUT(MaximumGrad) + +DECLARE_OP_ADAPTER(MinimumGrad) +DECLARE_OP_USE_OUTPUT(MinimumGrad) + +DECLARE_OP_ADAPTER(RealDiv) +DECLARE_OP_USE_OUTPUT(RealDiv) + +DECLARE_OP_ADAPTER(BitwiseAnd) +DECLARE_OP_USE_OUTPUT(BitwiseAnd) + +DECLARE_OP_ADAPTER(BitwiseOr) +DECLARE_OP_USE_OUTPUT(BitwiseOr) + +DECLARE_OP_ADAPTER(BitwiseXor) +DECLARE_OP_USE_OUTPUT(BitwiseXor) + +DECLARE_OP_ADAPTER(Rint) +DECLARE_OP_USE_OUTPUT(Rint) + +DECLARE_OP_ADAPTER(BesselI0e) +DECLARE_OP_USE_OUTPUT(BesselI0e) + +DECLARE_OP_ADAPTER(BesselI1e) +DECLARE_OP_USE_OUTPUT(BesselI1e) + +DECLARE_OP_ADAPTER(Inv) +DECLARE_OP_USE_OUTPUT(Inv) + +DECLARE_OP_ADAPTER(InvGrad) +DECLARE_OP_USE_OUTPUT(InvGrad) + +DECLARE_OP_ADAPTER(Invert) +DECLARE_OP_USE_OUTPUT(Invert) + +DECLARE_OP_ADAPTER(Log1p) +DECLARE_OP_USE_OUTPUT(Log1p) + +DECLARE_OP_ADAPTER(Ceil) +DECLARE_OP_USE_OUTPUT(Ceil) + +DECLARE_OP_ADAPTER(CosineEmbeddingLoss) +DECLARE_OP_USE_OUTPUT(CosineEmbeddingLoss) + +DECLARE_OP_ADAPTER(Xdivy) +DECLARE_OP_USE_OUTPUT(Xdivy) + +DECLARE_OP_ADAPTER(Mod) +DECLARE_OP_USE_OUTPUT(Mod) + +DECLARE_OP_ADAPTER(Cast) +DECLARE_OP_USE_INPUT_ATTR(Cast) +DECLARE_OP_USE_OUTPUT(Cast) + +DECLARE_OP_ADAPTER(Reciprocal) +DECLARE_OP_USE_OUTPUT(Reciprocal) + +DECLARE_OP_ADAPTER(Neg) +DECLARE_OP_USE_OUTPUT(Neg) + +DECLARE_OP_ADAPTER(Sub) +DECLARE_OP_USE_OUTPUT(Sub) + +DECLARE_OP_ADAPTER(Pow) +DECLARE_OP_USE_OUTPUT(Pow) + +DECLARE_OP_ADAPTER(PopulationCount) +DECLARE_OP_USE_OUTPUT(PopulationCount) + +DECLARE_OP_ADAPTER(Equal) +DECLARE_OP_USE_OUTPUT(Equal) + +DECLARE_OP_ADAPTER(ApproximateEqual) +DECLARE_OP_USE_OUTPUT(ApproximateEqual) + +DECLARE_OP_ADAPTER(NotEqual) +DECLARE_OP_USE_OUTPUT(NotEqual) + +DECLARE_OP_ADAPTER(Log) +DECLARE_OP_USE_OUTPUT(Log) + +DECLARE_OP_ADAPTER(LogicalAnd) +DECLARE_OP_USE_OUTPUT(LogicalAnd) + +DECLARE_OP_ADAPTER(LogicalOr) +DECLARE_OP_USE_OUTPUT(LogicalOr) + +DECLARE_OP_ADAPTER(LogicalNot) +DECLARE_OP_USE_OUTPUT(LogicalNot) + +DECLARE_OP_ADAPTER(LessEqual) +DECLARE_OP_USE_OUTPUT(LessEqual) + +DECLARE_OP_ADAPTER(Assign) +DECLARE_OP_USE_OUTPUT(Assign) + +DECLARE_OP_ADAPTER(Add) +DECLARE_OP_USE_OUTPUT(Add) + +DECLARE_OP_ADAPTER(Cos) +DECLARE_OP_USE_OUTPUT(Cos) + +DECLARE_OP_ADAPTER(Cosh) +DECLARE_OP_USE_OUTPUT(Cosh) + +DECLARE_OP_ADAPTER(Acos) +DECLARE_OP_USE_OUTPUT(Acos) + +DECLARE_OP_ADAPTER(AcosGrad) +DECLARE_OP_USE_OUTPUT(AcosGrad) + +DECLARE_OP_ADAPTER(Acosh) +DECLARE_OP_USE_OUTPUT(Acosh) + +DECLARE_OP_ADAPTER(AcoshGrad) +DECLARE_OP_USE_OUTPUT(AcoshGrad) + +DECLARE_OP_ADAPTER(Div) +DECLARE_OP_USE_OUTPUT(Div) + +DECLARE_OP_ADAPTER(TruncateDiv) +DECLARE_OP_USE_OUTPUT(TruncateDiv) + +DECLARE_OP_ADAPTER(TruncateMod) +DECLARE_OP_USE_OUTPUT(TruncateMod) + +DECLARE_OP_ADAPTER(Xlogy) +DECLARE_OP_USE_OUTPUT(Xlogy) + +DECLARE_OP_ADAPTER(DivNoNan) +DECLARE_OP_USE_OUTPUT(DivNoNan) + +DECLARE_OP_ADAPTER(Floor) +DECLARE_OP_USE_OUTPUT(Floor) + +DECLARE_OP_ADAPTER(FloorDiv) +DECLARE_OP_USE_OUTPUT(FloorDiv) + +DECLARE_OP_ADAPTER(FloorMod) +DECLARE_OP_USE_OUTPUT(FloorMod) + +DECLARE_OP_ADAPTER(Sin) +DECLARE_OP_USE_OUTPUT(Sin) + +DECLARE_OP_ADAPTER(Sinh) +DECLARE_OP_USE_OUTPUT(Sinh) + +DECLARE_OP_ADAPTER(Asin) +DECLARE_OP_USE_OUTPUT(Asin) + +DECLARE_OP_ADAPTER(AsinGrad) +DECLARE_OP_USE_OUTPUT(AsinGrad) + +DECLARE_OP_ADAPTER(Asinh) +DECLARE_OP_USE_OUTPUT(Asinh) + +DECLARE_OP_ADAPTER(AsinhGrad) +DECLARE_OP_USE_OUTPUT(AsinhGrad) + +DECLARE_OP_ADAPTER(Exp) +DECLARE_OP_USE_OUTPUT(Exp) + +DECLARE_OP_ADAPTER(Expm1) +DECLARE_OP_USE_OUTPUT(Expm1) + +DECLARE_OP_ADAPTER(BiasAdd) +DECLARE_OP_USE_OUTPUT(BiasAdd) + +DECLARE_OP_ADAPTER(Greater) +DECLARE_OP_USE_OUTPUT(Greater) + +DECLARE_OP_ADAPTER(SqrtGrad) +DECLARE_OP_USE_OUTPUT(SqrtGrad) + +DECLARE_OP_ADAPTER(ReciprocalGrad) +DECLARE_OP_USE_OUTPUT(ReciprocalGrad) + +DECLARE_OP_ADAPTER(RsqrtGrad) +DECLARE_OP_USE_OUTPUT(RsqrtGrad) + +DECLARE_OP_ADAPTER(Abs) +DECLARE_OP_USE_OUTPUT(Abs) + +DECLARE_OP_ADAPTER(AbsGrad) +DECLARE_OP_USE_OUTPUT(AbsGrad) + +DECLARE_OP_ADAPTER(Sign) +DECLARE_OP_USE_OUTPUT(Sign) + +DECLARE_OP_ADAPTER(Round) +DECLARE_OP_USE_OUTPUT(Round) + +DECLARE_OP_ADAPTER(Tan) +DECLARE_OP_USE_OUTPUT(Tan) + +DECLARE_OP_ADAPTER(Atan) +DECLARE_OP_USE_OUTPUT(Atan) + +DECLARE_OP_ADAPTER(AtanGrad) +DECLARE_OP_USE_OUTPUT(AtanGrad) + +DECLARE_OP_ADAPTER(Atanh) +DECLARE_OP_USE_OUTPUT(Atanh) + +DECLARE_OP_ADAPTER(Atan2) +DECLARE_OP_USE_OUTPUT(Atan2) + +DECLARE_OP_ADAPTER(LambApplyOptimizerAssign) +DECLARE_OP_USE_OUTPUT(LambApplyOptimizerAssign) + +DECLARE_OP_ADAPTER(LambApplyWeightAssign) +DECLARE_OP_USE_OUTPUT(LambApplyWeightAssign) + +DECLARE_OP_ADAPTER(Eltwise) +DECLARE_OP_USE_OUTPUT(Eltwise) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_ELEWISE_CALCULATION_OPS_DECLARE_H_ -- 2.34.1 From f9a72796c276ed1f0a9b692e9e5f60f1ae155746 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:37:45 +0800 Subject: [PATCH 020/160] ADD file via upload --- .../graph_ir/functional_ops_declare.cc | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.cc new file mode 100644 index 00000000000..6a5c3b053e8 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.cc @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/functional_ops_declare.h" + +namespace mindspore::transform { +// Case +INPUT_MAP(Case) = {{1, INPUT_DESC(branch_index)}};/* + 将branch_index处理并存入对应INPUT_DESC结构体的相应变量中 +  将branch_index内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将Case的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +DYN_INPUT_MAP(Case) = {{2, DYN_INPUT_DESC(input)}};/* + 将branch_index处理并存入对应DYN_INPUT_DESC结构体的相应变量中 +  将branch_index内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为DYN_INPUT_DESC结构体 +将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容 +*/ +ATTR_MAP(Case) = EMPTY_ATTR_MAP;//将Case与标准进行比较，使原以attr_map_为储存信息的key变为空 +DYN_OUTPUT_MAP(Case) = {{0, DYN_OUTPUT_DESC(output)}};/* + 将(output处理并存入对应DYN_OUTPUT_DESC结构体的相应变量中 +  将(output内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为DYN_OUTPUT_DESC结构体 +将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_output_map_为key存储相应内容 +*/ +DYN_SUBGRAPH_MAP(Case) = {{0, DYN_SUBGRAPH_DESC(branches)}};/* + 将branches处理并存入对应DYN_SUBGRAPH_DESC结构体的相应变量中 +  将branches内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为DYN_SUBGRAPH_DESC结构体 +将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_subgrraph_map_为key存储相应内容 +*/ +REG_ADPT_DESC(Case, kNameCase, ADPT_DESC(Case))/* + 将Case处理并存入对应ADPT_DESC结构体的相应变量中 +  将Case内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将Case处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将Case内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ +} // namespace mindspore::transform -- 2.34.1 From fbde8ca706be362c8e5c04aaa5e0fc84f8f0ed3e Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:38:01 +0800 Subject: [PATCH 021/160] ADD file via upload --- .../graph_ir/functional_ops_declare.h | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.h new file mode 100644 index 00000000000..476385ec5b3 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/functional_ops_declare.h @@ -0,0 +1,31 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_FUNCTIONAL_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_FUNCTIONAL_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/functional_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(Case)//将Case的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_DYN_INPUT(Case)//将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容 +DECLARE_OP_USE_DYN_SUBGRAPH(Case)//将ACase的类型与标准进行对比，后进行空间调整，并用指针dyn_subgraph__map_为key存储相应内容 +DECLARE_OP_USE_DYN_OUTPUT(Case)//将Case的类型与标准进行对比，后进行空间调整，并用指针dyn_output_map_为key存储相应内容 +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_FUNCTIONAL_OPS_DECLARE_H_ -- 2.34.1 From b594519341096c8d6bf9a87a4243eacacb7155e0 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:38:25 +0800 Subject: [PATCH 022/160] ADD file via upload --- .../transform/graph_ir/hcom_ops_declare.cc | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.cc new file mode 100644 index 00000000000..f08f4e027c3 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.cc @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/hcom_ops_declare.h" + +namespace mindspore::transform { +// HCOMAllreduce +INPUT_MAP(HcomAllReduce) = {{1, INPUT_DESC(x)}};/* + 将x处理并存入对应INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将HcomAllReduce的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +OUTPUT_MAP(HcomAllReduce) = {{0, OUTPUT_DESC(y)}};/* + 将y处理并存入对应OUTPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体 +将(HcomAllReduce的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ +ATTR_MAP(HcomAllReduce) = {{"op", ATTR_DESC(reduction, AnyTraits())}, + {"group", ATTR_DESC(group, AnyTraits())}, + {"fusion", ATTR_DESC(fusion, AnyTraits())}};/* + 将reduction处理并存入对应ATTR_DESC结构体的相应变量中 +  将reduction内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +将HcomAllReduce的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +REG_ADPT_DESC(HcomAllReduce, kNameAllReduce, ADPT_DESC(HcomAllReduce))/* + 将HcomAllReduce处理并存入对应ADPT_DESC结构体的相应变量中 +  将HcomAllReduce内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将HcomAllReduce处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将HcomAllReduce内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +// HCOMBraodcast +INPUT_MAP(HcomBroadcast) = EMPTY_INPUT_MAP; +DYN_INPUT_MAP(HcomBroadcast) = {{1, DYN_INPUT_DESC(x)}}; +DYN_OUTPUT_MAP(HcomBroadcast) = {{0, DYN_OUTPUT_DESC(y)}}; +ATTR_MAP(HcomBroadcast) = {{"root_rank", ATTR_DESC(root_rank, AnyTraits())}, + {"group", ATTR_DESC(group, AnyTraits())}}; +REG_ADPT_DESC(HcomBroadcast, kNameBroadcast, ADPT_DESC(HcomBroadcast)) + +// HcomAllGather +INPUT_MAP(HcomAllGather) = {{1, INPUT_DESC(x)}}; +OUTPUT_MAP(HcomAllGather) = {{0, OUTPUT_DESC(y)}}; +ATTR_MAP(HcomAllGather) = {{"group", ATTR_DESC(group, AnyTraits())}, + {"rank_size", ATTR_DESC(rank_size, AnyTraits())}}; +REG_ADPT_DESC(HcomAllGather, kNameAllgather, ADPT_DESC(HcomAllGather)) + +// HCOMReduceScatter +INPUT_MAP(HcomReduceScatter) = {{1, INPUT_DESC(x)}}; +OUTPUT_MAP(HcomReduceScatter) = {{0, OUTPUT_DESC(y)}}; +ATTR_MAP(HcomReduceScatter) = {{"group", ATTR_DESC(group, AnyTraits())}, + {"op", ATTR_DESC(reduction, AnyTraits())}, + {"rank_size", ATTR_DESC(rank_size, AnyTraits())}}; +REG_ADPT_DESC(HcomReduceScatter, kNameReduceScatter, ADPT_DESC(HcomReduceScatter)) +} // namespace mindspore::transform -- 2.34.1 From 14b3ebadb6008ddca899cfdd1598f2e320db537f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:38:43 +0800 Subject: [PATCH 023/160] ADD file via upload --- .../transform/graph_ir/hcom_ops_declare.h | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.h new file mode 100644 index 00000000000..59cc0cb1d12 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/hcom_ops_declare.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_HCOM_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_HCOM_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/hcom_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(HcomReduceScatter)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(HcomReduceScatter)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 + +DECLARE_OP_ADAPTER(HcomBroadcast) +DECLARE_OP_USE_DYN_INPUT(HcomBroadcast)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容 +DECLARE_OP_USE_DYN_OUTPUT(HcomBroadcast)//将HcomReduceScatter的类型与标准进行对比，后进行空间调整，并用指针dyn_output_map_为key存储相应内容 + +DECLARE_OP_ADAPTER(HcomAllReduce) +DECLARE_OP_USE_OUTPUT(HcomAllReduce) + +DECLARE_OP_ADAPTER(HcomAllGather) +DECLARE_OP_USE_OUTPUT(HcomAllGather) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_HCOM_OPS_DECLARE_H_ -- 2.34.1 From dd6cc3c280f46c52ce20e154bd1b845ee37cadab Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:39:02 +0800 Subject: [PATCH 024/160] Delete 'mindspore2022/ccsrc/transform/graph_ir/op_declare' --- mindspore2022/ccsrc/transform/graph_ir/op_declare | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_declare diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_declare b/mindspore2022/ccsrc/transform/graph_ir/op_declare deleted file mode 100644 index e69de29bb2d..00000000000 -- 2.34.1 From 7cff8311c33802f05523cee1d203cdf3833efe37 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:40:11 +0800 Subject: [PATCH 025/160] ADD file via upload --- .../transform/graph_ir/image_ops_declare.cc | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.cc new file mode 100644 index 00000000000..0bbcd9e0c7e --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.cc @@ -0,0 +1,90 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/image_ops_declare.h" +#include + +namespace mindspore::transform { +// ResizeNearestNeighborV2D +INPUT_MAP(ResizeNearestNeighborV2D) = {{1, INPUT_DESC(x)}};/* + 将x处理并存入对应INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将ResizeNearestNeighborV2D的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +ATTR_MAP(ResizeNearestNeighborV2D) = { + {"size", ATTR_DESC(size, AnyTraits>(), AnyTraits>())}, + {"align_corners", ATTR_DESC(align_corners, AnyTraits())}};/* + 将size处理并存入对应ATTR_DESC结构体的相应变量中 +  将size内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +将ResizeNearestNeighborV2D的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(ResizeNearestNeighborV2D) = {{0, OUTPUT_DESC(y)}};/* + 将y处理并存入对应OUTPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体 +将ResizeNearestNeighborV2D的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ +REG_ADPT_DESC(ResizeNearestNeighborV2D, kNameResizeNearestNeighborD, ADPT_DESC(ResizeNearestNeighborV2D))/* + 将ResizeNearestNeighborV2D处理并存入对应ADPT_DESC结构体的相应变量中 +  将ResizeNearestNeighborV2D内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将ResizeNearestNeighborV2D处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将ResizeNearestNeighborV2D内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +// ResizeNearestNeighborV2 +INPUT_MAP(ResizeNearestNeighborV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(size)}}; +ATTR_MAP(ResizeNearestNeighborV2) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}, + {"half_pixel_centers", ATTR_DESC(half_pixel_centers, AnyTraits())}}; +OUTPUT_MAP(ResizeNearestNeighborV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ResizeNearestNeighborV2, kNameResizeNearestNeighborV2, ADPT_DESC(ResizeNearestNeighborV2)) + +// ResizeNearestNeighborV2Grad +INPUT_MAP(ResizeNearestNeighborV2Grad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(size)}}; +ATTR_MAP(ResizeNearestNeighborV2Grad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeNearestNeighborV2Grad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ResizeNearestNeighborV2Grad, kNameResizeNearestNeighborGrad, ADPT_DESC(ResizeNearestNeighborV2Grad)) + +// ResizeBilinearV2Grad +INPUT_MAP(ResizeBilinearV2Grad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(original_image)}}; +ATTR_MAP(ResizeBilinearV2Grad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeBilinearV2Grad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ResizeBilinearV2Grad, kNameResizeBilinearGrad, ADPT_DESC(ResizeBilinearV2Grad)) + +// ResizeBilinearV2D +INPUT_MAP(ResizeBilinearV2D) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(ResizeBilinearV2D) = { + {"size", ATTR_DESC(size, AnyTraits>(), AnyTraits>())}, + {"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +OUTPUT_MAP(ResizeBilinearV2D) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ResizeBilinearV2D, kNameResizeBilinear, ADPT_DESC(ResizeBilinearV2D)) + +// CropAndResize +INPUT_MAP(CropAndResize) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(boxes)}, {3, INPUT_DESC(box_index)}, {4, INPUT_DESC(crop_size)}}; +ATTR_MAP(CropAndResize) = {{"extrapolation_value", ATTR_DESC(extrapolation_value, AnyTraits())}, + {"method", ATTR_DESC(method, AnyTraits())}}; +OUTPUT_MAP(CropAndResize) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(CropAndResize, kNameCropAndResize, ADPT_DESC(CropAndResize)) +} // namespace mindspore::transform -- 2.34.1 From a002adb55a1d6d19e0a78bf001e79a3a9d2336b8 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:40:30 +0800 Subject: [PATCH 026/160] ADD file via upload --- .../transform/graph_ir/image_ops_declare.h | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.h new file mode 100644 index 00000000000..ce1be47451c --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/image_ops_declare.h @@ -0,0 +1,44 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/image_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(ResizeNearestNeighborV2D)//将ResizeNearestNeighborV2D收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborV2D)//将ResizeNearestNeighborV2D收录后与标准进行比较，进行空间调整，用output_map_为key储存 + +DECLARE_OP_ADAPTER(ResizeNearestNeighborV2) +DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborV2) + +DECLARE_OP_ADAPTER(ResizeNearestNeighborV2Grad) +DECLARE_OP_USE_OUTPUT(ResizeNearestNeighborV2Grad) + +DECLARE_OP_ADAPTER(ResizeBilinearV2D) +DECLARE_OP_USE_OUTPUT(ResizeBilinearV2D) + +DECLARE_OP_ADAPTER(ResizeBilinearV2Grad) +DECLARE_OP_USE_OUTPUT(ResizeBilinearV2Grad) + +DECLARE_OP_ADAPTER(CropAndResize) +DECLARE_OP_USE_OUTPUT(CropAndResize) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_ -- 2.34.1 From e64ca6ab96e1ec4df0c262d8dcbc54767d8e6eaa Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:40:50 +0800 Subject: [PATCH 027/160] ADD file via upload --- .../transform/graph_ir/logging_ops_declare.cc | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.cc new file mode 100644 index 00000000000..ace6511a4d8 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/logging_ops_declare.h" + +namespace mindspore::transform { +// Print +INPUT_MAP(Print) = EMPTY_INPUT_MAP;//将Print的类型与标准进行对比，后进行空间调整，使原input_map_为储存相应内容的key为空 +DYN_INPUT_MAP(Print) = {{1, DYN_INPUT_DESC(x)}};/* + 将x处理并存入对应DYN_INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为DYN_INPUT_DESC结构体 +将Print的类型与标准进行对比，后进行空间调整，并用指针dyn_input_map_为key存储相应内容 +*/ +ATTR_MAP(Print) = EMPTY_ATTR_MAP;//将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，使原attr_map_为储存相应内容的key为空 +REG_ADPT_DESC(Print, kNamePrint, ADPT_DESC(Print))/* + 将Print处理并存入对应ADPT_DESC结构体的相应变量中 +  将Print内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将Print处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将Print内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + + +#ifdef ENABLE_D +INPUT_MAP(Assert) = {{1, INPUT_DESC(input_condition)}}; +DYN_INPUT_MAP(Assert) = {{2, DYN_INPUT_DESC(input_data)}}; +ATTR_MAP(Assert) = {{"summarize", ATTR_DESC(summarize, AnyTraits())}}; +REG_ADPT_DESC(Assert, kNameAssert, ADPT_DESC(Assert)) +#endif +} // namespace mindspore::transform -- 2.34.1 From 19baf8b339a6a9f2d194e88c07901c7160c881ae Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:41:14 +0800 Subject: [PATCH 028/160] ADD file via upload --- .../transform/graph_ir/logging_ops_declare.h | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.h new file mode 100644 index 00000000000..0a633dabca4 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/logging_ops_declare.h @@ -0,0 +1,34 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_LOGGING_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_LOGGING_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/logging_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(Print)//将Print收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_DYN_INPUT(Print)//将Print收录后与标准进行比较，进行空间调整，用dyn_input_map_为key储存 + +#ifdef ENABLE_D +DECLARE_OP_ADAPTER(Assert) +DECLARE_OP_USE_DYN_INPUT(Assert) +#endif +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_LOGGING_OPS_DECLARE_H_ -- 2.34.1 From e3fbbb1becd45ec3525e8f1c2817ce7a6c484bb5 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:41:33 +0800 Subject: [PATCH 029/160] ADD file via upload --- .../transform/graph_ir/math_ops_declare.cc | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.cc new file mode 100644 index 00000000000..3a74b76f0f4 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.cc @@ -0,0 +1,132 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/math_ops_declare.h" +#include + +namespace mindspore::transform { +// ActsULQ +INPUT_MAP(ActsULQ) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(clamp_min)}, {3, INPUT_DESC(clamp_max)}}; +//以其中一句为例 +/* + 将x处理并存入对应INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将ActsULQ的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +ATTR_MAP(ActsULQ) = {{"fixed_min", ATTR_DESC(fixed_min, AnyTraits())}, + {"num_bits", ATTR_DESC(num_bits, AnyTraits())}}; + /* + 将fixed_min处理并存入对应ATTR_DESC结构体的相应变量中 +  将fixed_min内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +将ActsULQ的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(ActsULQ) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(clamp_min_mask)}, + {2, OUTPUT_DESC(clamp_max_mask)}, + {3, OUTPUT_DESC(x_clamped_loss)}};/* + 将y处理并存入对应INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将ActsULQ的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ +REG_ADPT_DESC(ActsULQ, kNameActsULQ, ADPT_DESC(ActsULQ))/* + 将ActsULQ处理并存入对应ADPT_DESC结构体的相应变量中 +  将ActsULQ内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将ActsULQ处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将ActsULQ内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +// ActsULQInputGrad +INPUT_MAP(ActsULQInputGrad) = { + {1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_min_mask)}, {3, INPUT_DESC(clamp_max_mask)}}; +ATTR_MAP(ActsULQInputGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ActsULQInputGrad) = {{0, OUTPUT_DESC(x_grad)}}; +REG_ADPT_DESC(ActsULQInputGrad, kNameActsULQInputGrad, ADPT_DESC(ActsULQInputGrad)) + +// ActULQClampMaxGrad +INPUT_MAP(ActULQClampMaxGrad) = { + {1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_max_mask)}, {3, INPUT_DESC(x_clamped_loss)}}; +ATTR_MAP(ActULQClampMaxGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ActULQClampMaxGrad) = {{0, OUTPUT_DESC(clamp_max_grad)}}; +REG_ADPT_DESC(ActULQClampMaxGrad, kNameActULQClampMaxGrad, ADPT_DESC(ActULQClampMaxGrad)) + +// ActULQClampMinGrad +INPUT_MAP(ActULQClampMinGrad) = { + {1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_min_mask)}, {3, INPUT_DESC(x_clamped_loss)}}; +ATTR_MAP(ActULQClampMinGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(ActULQClampMinGrad) = {{0, OUTPUT_DESC(clamp_min_grad)}}; +REG_ADPT_DESC(ActULQClampMinGrad, kNameActULQClampMinGrad, ADPT_DESC(ActULQClampMinGrad)) + +// HistogramFixedWidthD +INPUT_MAP(HistogramFixedWidthD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(range)}}; +ATTR_MAP(HistogramFixedWidthD) = {{"nbins", ATTR_DESC(nbins, AnyTraits())}, + {"dtype", ATTR_DESC(dtype, AnyTraits())}}; +OUTPUT_MAP(HistogramFixedWidthD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(HistogramFixedWidthD, kNameHistogramFixedWidthD, ADPT_DESC(HistogramFixedWidthD)) + +// IFMR +INPUT_MAP(IFMR) = { + {1, INPUT_DESC(data)}, {2, INPUT_DESC(data_min)}, {3, INPUT_DESC(data_max)}, {4, INPUT_DESC(cumsum)}}; +ATTR_MAP(IFMR) = {{"min_percentile", ATTR_DESC(min_percentile, AnyTraits())}, + {"max_percentile", ATTR_DESC(max_percentile, AnyTraits())}, + {"search_range", ATTR_DESC(search_range, AnyTraits>())}, + {"search_step", ATTR_DESC(search_step, AnyTraits())}}; +OUTPUT_MAP(IFMR) = {{0, OUTPUT_DESC(scale)}, {1, OUTPUT_DESC(offset)}}; +REG_ADPT_DESC(IFMR, kNameIFMR, ADPT_DESC(IFMR)) + +// NLLLoss +INPUT_MAP(NLLLoss) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(weight)}}; +ATTR_MAP(NLLLoss) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(NLLLoss) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(total_weight)}}; +REG_ADPT_DESC(NLLLoss, kNameNLLLoss, ADPT_DESC(NLLLoss)) + +// NLLLossGrad +INPUT_MAP(NLLLossGrad) = {{1, INPUT_DESC(x)}, + {2, INPUT_DESC(y_grad)}, + {3, INPUT_DESC(target)}, + {4, INPUT_DESC(weight)}, + {5, INPUT_DESC(total_weight)}}; +ATTR_MAP(NLLLossGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(NLLLossGrad) = {{0, OUTPUT_DESC(x_grad)}}; +REG_ADPT_DESC(NLLLossGrad, kNameNLLLossGrad, ADPT_DESC(NLLLossGrad)) + +// Erf +INPUT_MAP(Erf) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Erf) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Erf) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Erf, kNameErf, ADPT_DESC(Erf)) + +// Erfc +INPUT_MAP(Erfc) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Erfc) = EMPTY_ATTR_MAP; +OUTPUT_MAP(Erfc) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Erfc, kNameErfc, ADPT_DESC(Erfc)) + +// WtsARQ +INPUT_MAP(WtsARQ) = {{1, INPUT_DESC(w)}, {2, INPUT_DESC(w_min)}, {3, INPUT_DESC(w_max)}}; +ATTR_MAP(WtsARQ) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits())}, + {"offset_flag", ATTR_DESC(offset_flag, AnyTraits())}}; +OUTPUT_MAP(WtsARQ) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(WtsARQ, kNameWtsARQ, ADPT_DESC(WtsARQ)) +} // namespace mindspore::transform -- 2.34.1 From d428e6a24c810cfc28b44274299b473fc8030ecd Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:43:11 +0800 Subject: [PATCH 030/160] ADD file via upload --- .../transform/graph_ir/math_ops_declare.h | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.h new file mode 100644 index 00000000000..edc19ac0749 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/math_ops_declare.h @@ -0,0 +1,59 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATH_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATH_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/math_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(ActsULQ)//将ActsULQ收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(ActsULQ)//将ActsULQ收录后与标准进行比较，进行空间调整，用output_map_为key储存 + +DECLARE_OP_ADAPTER(ActsULQInputGrad) +DECLARE_OP_USE_OUTPUT(ActsULQInputGrad) + +DECLARE_OP_ADAPTER(ActULQClampMaxGrad) +DECLARE_OP_USE_OUTPUT(ActULQClampMaxGrad) + +DECLARE_OP_ADAPTER(ActULQClampMinGrad) +DECLARE_OP_USE_OUTPUT(ActULQClampMinGrad) + +DECLARE_OP_ADAPTER(HistogramFixedWidthD) +DECLARE_OP_USE_OUTPUT(HistogramFixedWidthD) + +DECLARE_OP_ADAPTER(IFMR) +DECLARE_OP_USE_OUTPUT(IFMR) + +DECLARE_OP_ADAPTER(NLLLoss) +DECLARE_OP_USE_OUTPUT(NLLLoss) + +DECLARE_OP_ADAPTER(NLLLossGrad) +DECLARE_OP_USE_OUTPUT(NLLLossGrad) + +DECLARE_OP_ADAPTER(Erf) +DECLARE_OP_USE_OUTPUT(Erf) + +DECLARE_OP_ADAPTER(Erfc) +DECLARE_OP_USE_OUTPUT(Erfc) + +DECLARE_OP_ADAPTER(WtsARQ) +DECLARE_OP_USE_OUTPUT(WtsARQ) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATH_OPS_DECLARE_H_ -- 2.34.1 From 1201865497f00322fac517fc330ef61d436512fb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:43:28 +0800 Subject: [PATCH 031/160] ADD file via upload --- .../matrix_calculation_ops_declare.cc | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.cc new file mode 100644 index 00000000000..f69ac4976a5 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.cc @@ -0,0 +1,180 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/matrix_calculation_ops_declare.h" + +namespace mindspore::transform { +// TensorScatterUpdate +INPUT_MAP(TensorScatterUpdate) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +//以其中一句为例 +/* + 将x处理并存入对应INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +ATTR_MAP(TensorScatterUpdate) = EMPTY_ATTR_MAP;//将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，使原attr_map_为储存相应内容的key为空 +OUTPUT_MAP(TensorScatterUpdate) = {{0, OUTPUT_DESC(y)}};/* + 将y处理并存入对应OUTPUT_DESC结构体的相应变量中 +  将y内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将TensorScatterUpdate的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ +REG_ADPT_DESC(TensorScatterUpdate, kNameTensorScatterUpdate, ADPT_DESC(TensorScatterUpdate))/* + 将TensorScatterUpdate处理并存入对应ADPT_DESC结构体的相应变量中 +  将TensorScatterUpdate内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将TensorScatterUpdate处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将TensorScatterUpdate内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +// ScatterUpdate +INPUT_MAP(ScatterUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterUpdate) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterUpdate, kNameScatterUpdate, ADPT_DESC(ScatterUpdate)) + +// ScatterNdUpdate +INPUT_MAP(ScatterNdUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdUpdate) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterNdUpdate, kNameScatterNdUpdate, ADPT_DESC(ScatterNdUpdate)) + +// ScatterMax +INPUT_MAP(ScatterMax) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterMax) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterMax) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterMax, kNameScatterMax, ADPT_DESC(ScatterMax)) + +// ScatterMin +INPUT_MAP(ScatterMin) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterMin) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterMin) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterMin, kNameScatterMin, ADPT_DESC(ScatterMin)) + +// ScatterAdd +INPUT_MAP(ScatterAdd) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterAdd) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterAdd) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterAdd, kNameScatterAdd, ADPT_DESC(ScatterAdd)) + +// ScatterSub +INPUT_MAP(ScatterSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterSub) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterSub) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterSub, kNameScatterSub, ADPT_DESC(ScatterSub)) + +// ScatterMul +INPUT_MAP(ScatterMul) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterMul) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterMul) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterMul, kNameScatterMul, ADPT_DESC(ScatterMul)) + +// ScatterDiv +INPUT_MAP(ScatterDiv) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterDiv) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterDiv) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterDiv, kNameScatterDiv, ADPT_DESC(ScatterDiv)) + +// ScatterNdAdd +INPUT_MAP(ScatterNdAdd) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdAdd) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdAdd) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterNdAdd, kNameScatterNdAdd, ADPT_DESC(ScatterNdAdd)) + +// ScatterNdSub +INPUT_MAP(ScatterNdSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdSub) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdSub) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterNdSub, kNameScatterNdSub, ADPT_DESC(ScatterNdSub)) + +// MatMul +INPUT_MAP(MatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(MatMul) = {{"transpose_x1", ATTR_DESC(transpose_x1, AnyTraits())}, + {"transpose_x2", ATTR_DESC(transpose_x2, AnyTraits())}}; +OUTPUT_MAP(MatMul) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatMul, kNameMatMul, ADPT_DESC(MatMul)) + +// MatMulV2 +INPUT_MAP(MatMulV2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(MatMulV2) = {{"transpose_a", ATTR_DESC(transpose_x1, AnyTraits())}, + {"transpose_b", ATTR_DESC(transpose_x2, AnyTraits())}}; +OUTPUT_MAP(MatMulV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatMulV2, prim::kPrimMatMul->name(), ADPT_DESC(MatMulV2)) + +// MatrixDiagD +INPUT_MAP(MatrixDiagD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(assist)}}; +ATTR_MAP(MatrixDiagD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MatrixDiagD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatrixDiagD, kNameMatrixDiagD, ADPT_DESC(MatrixDiagD)) + +// MatrixDiagPartD +INPUT_MAP(MatrixDiagPartD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(assist)}}; +ATTR_MAP(MatrixDiagPartD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MatrixDiagPartD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatrixDiagPartD, kNameMatrixDiagPartD, ADPT_DESC(MatrixDiagPartD)) + +// MatrixSetDiagD +INPUT_MAP(MatrixSetDiagD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(diagonal)}, {3, INPUT_DESC(assist)}}; +ATTR_MAP(MatrixSetDiagD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MatrixSetDiagD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatrixSetDiagD, kNameMatrixSetDiagD, ADPT_DESC(MatrixSetDiagD)) + +// DiagPart +INPUT_MAP(DiagPart) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(DiagPart) = EMPTY_ATTR_MAP; +OUTPUT_MAP(DiagPart) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(DiagPart, kNameDiagPart, ADPT_DESC(DiagPart)) + +// BatchMatMul +INPUT_MAP(BatchMatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BatchMatMul) = {{"transpose_x1", ATTR_DESC(adj_x1, AnyTraits())}, + {"transpose_x2", ATTR_DESC(adj_x2, AnyTraits())}}; +OUTPUT_MAP(BatchMatMul) = {{0, OUTPUT_DESC(y)}}; + +// BatchMatMul->BatchMatMulV2 +INPUT_MAP(BatchMatMulV2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BatchMatMulV2) = {{"transpose_x1", ATTR_DESC(adj_x1, AnyTraits())}, + {"transpose_x2", ATTR_DESC(adj_x2, AnyTraits())}}; +OUTPUT_MAP(BatchMatMulV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BatchMatMul, kNameBatchMatMul, ADPT_DESC(BatchMatMul)) +REG_ADPT_DESC(BatchMatMulV2, kNameBatchMatMulV2, ADPT_DESC(BatchMatMulV2)) + +// L2Loss +INPUT_MAP(L2Loss) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(L2Loss) = EMPTY_ATTR_MAP; +OUTPUT_MAP(L2Loss) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(L2Loss, kNameL2Loss, ADPT_DESC(L2Loss)) + +// ScatterElements +INPUT_MAP(ScatterElements) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterElements) = {{"axis", ATTR_DESC(axis, AnyTraits())}}; +OUTPUT_MAP(ScatterElements) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ScatterElements, kNameScatterElements, ADPT_DESC(ScatterElements)) + +// FullyConnection +INPUT_MAP(FullyConnection) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, {4, INPUT_DESC(offset_w)}}; + +ATTR_MAP(FullyConnection) = {{"num_output", ATTR_DESC(num_output, AnyTraits())}, + {"transpose", ATTR_DESC(transpose, AnyTraits())}, + {"axis", ATTR_DESC(axis, AnyTraits())}, + {"offset_x", ATTR_DESC(offset_x, AnyTraits())}}; + +OUTPUT_MAP(FullyConnection) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(FullyConnection, kNameFullConnection, ADPT_DESC(FullyConnection)) +} // namespace mindspore::transform -- 2.34.1 From fba9822b141b0af2ed6124f6861e6133b20a2212 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:43:45 +0800 Subject: [PATCH 032/160] ADD file via upload --- .../graph_ir/matrix_calculation_ops_declare.h | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.h new file mode 100644 index 00000000000..8bad02f70a4 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/matrix_calculation_ops_declare.h @@ -0,0 +1,92 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATRIX_CALCULATION_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATRIX_CALCULATION_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/matrix_calculation_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(TensorScatterUpdate)//将TensorScatterUpdate收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(TensorScatterUpdate)//将TensorScatterUpdate收录后与标准进行比较，进行空间调整，用output_map_为key储存 + +DECLARE_OP_ADAPTER(ScatterUpdate) +DECLARE_OP_USE_OUTPUT(ScatterUpdate) + +DECLARE_OP_ADAPTER(ScatterNdUpdate) +DECLARE_OP_USE_OUTPUT(ScatterNdUpdate) + +DECLARE_OP_ADAPTER(ScatterMax) +DECLARE_OP_USE_OUTPUT(ScatterMax) + +DECLARE_OP_ADAPTER(ScatterMin) +DECLARE_OP_USE_OUTPUT(ScatterMin) + +DECLARE_OP_ADAPTER(ScatterAdd) +DECLARE_OP_USE_OUTPUT(ScatterAdd) + +DECLARE_OP_ADAPTER(ScatterSub) +DECLARE_OP_USE_OUTPUT(ScatterSub) + +DECLARE_OP_ADAPTER(ScatterMul) +DECLARE_OP_USE_OUTPUT(ScatterMul) + +DECLARE_OP_ADAPTER(ScatterDiv) +DECLARE_OP_USE_OUTPUT(ScatterDiv) + +DECLARE_OP_ADAPTER(ScatterNdAdd) +DECLARE_OP_USE_OUTPUT(ScatterNdAdd) + +DECLARE_OP_ADAPTER(ScatterNdSub) +DECLARE_OP_USE_OUTPUT(ScatterNdSub) + +DECLARE_OP_ADAPTER(BatchMatMul) +DECLARE_OP_USE_OUTPUT(BatchMatMul) + +DECLARE_OP_ADAPTER(BatchMatMulV2) +DECLARE_OP_USE_OUTPUT(BatchMatMulV2) + +DECLARE_OP_ADAPTER(MatMul) +DECLARE_OP_USE_OUTPUT(MatMul) + +DECLARE_OP_ADAPTER(MatMulV2) +DECLARE_OP_USE_OUTPUT(MatMulV2) + +DECLARE_OP_ADAPTER(MatrixDiagD) +DECLARE_OP_USE_OUTPUT(MatrixDiagD) + +DECLARE_OP_ADAPTER(MatrixDiagPartD) +DECLARE_OP_USE_OUTPUT(MatrixDiagPartD) + +DECLARE_OP_ADAPTER(MatrixSetDiagD) +DECLARE_OP_USE_OUTPUT(MatrixSetDiagD) + +DECLARE_OP_ADAPTER(DiagPart) +DECLARE_OP_USE_OUTPUT(DiagPart) + +DECLARE_OP_ADAPTER(L2Loss) +DECLARE_OP_USE_OUTPUT(L2Loss) + +DECLARE_OP_ADAPTER(ScatterElements) +DECLARE_OP_USE_OUTPUT(ScatterElements) + +DECLARE_OP_ADAPTER(FullyConnection) +DECLARE_OP_USE_OUTPUT(FullyConnection) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MATRIX_CALCULATION_OPS_DECLARE_H_ -- 2.34.1 From e9ac28e746de69dec3ac1bd2c1754e6b0e055304 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:44:07 +0800 Subject: [PATCH 033/160] ADD file via upload --- .../graph_ir/nn_batch_norm_ops_declare.cc | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.cc new file mode 100644 index 00000000000..58cf5791eec --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.cc @@ -0,0 +1,109 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_batch_norm_ops_declare.h" +#include + +namespace mindspore::transform { +// BatchNorm +INPUT_MAP(BatchNorm) = {{1, INPUT_DESC(x)}, + {2, INPUT_DESC(scale)}, + {3, INPUT_DESC(offset)}, + {4, INPUT_DESC(mean)}, + {5, INPUT_DESC(variance)}}; +ATTR_MAP(BatchNorm) = {{"format", ATTR_DESC(data_format, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +OUTPUT_MAP(BatchNorm) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(batch_mean)}, + {2, OUTPUT_DESC(batch_variance)}, + {3, OUTPUT_DESC(reserve_space_1)}, + {4, OUTPUT_DESC(reserve_space_2)}}; +// BNInference is BatchNorm for caffe +//用这一部分做注释，宏定义用的比较全 +INPUT_MAP(BNInference) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mean)}, {3, INPUT_DESC(variance)}, + {4, INPUT_DESC(momentum)}, {5, INPUT_DESC(scale)}, {6, INPUT_DESC(offset)}}; +//以其中一句为例 +/* + 将x处理并存入对应INPUT_DESC结构体的相应变量中 +  将x内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将BNInference的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +ATTR_MAP(BNInference) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"use_global_stats", ATTR_DESC(use_global_stats, AnyTraits())}, + {"mode", ATTR_DESC(mode, AnyTraits())}}; +/* + 将epsilon处理并存入对应ATTR_DESC结构体的相应变量中 +  将epsilon内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +将BNInference的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ +OUTPUT_MAP(BNInference) = {{0, OUTPUT_DESC(y)}};/* + 将y处理并存入对应OUTPUT_DESC结构体的相应变量中 +  将y内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为INPUT_DESC结构体 +将BNInference的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ + +REG_ADPT_DESC(BNInference, kNameBNInference, ADPT_DESC(BNInference))/* + 将NInference处理并存入对应ADPT_DESC结构体的相应变量中 +  将NInference内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将NInference处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将NInference内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ +REG_ADPT_DESC(BatchNorm, kNameBatchNorm, ADPT_DESC(BatchNorm)) +REG_ADPT_DESC(FusedBatchNorm, kNameFusedBatchNorm, ADPT_DESC(BatchNorm)) + +// BatchNormGrad +INPUT_MAP(BatchNormGrad) = {{1, INPUT_DESC(y_backprop)}, + {2, INPUT_DESC(x)}, + {3, INPUT_DESC(scale)}, + {4, INPUT_DESC(reserve_space_1)}, + {5, INPUT_DESC(reserve_space_2)}}; +ATTR_MAP(BatchNormGrad) = {{"format", ATTR_DESC(data_format, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +OUTPUT_MAP(BatchNormGrad) = {{0, OUTPUT_DESC(x_backprop)}, + {1, OUTPUT_DESC(scale_backprop)}, + {2, OUTPUT_DESC(offset_backprop)}, + {3, OUTPUT_DESC(reserve_space_4)}, + {4, OUTPUT_DESC(reserve_space_5)}}; +REG_ADPT_DESC(BatchNormGrad, kNameBatchNormGrad, ADPT_DESC(BatchNormGrad)) + +// L2NormalizeGrad +INPUT_MAP(L2NormalizeGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(dy)}}; +ATTR_MAP(L2NormalizeGrad) = { + {"axis", ATTR_DESC(dim, AnyTraits>(), AnyTraits>())}, + {"epsilon", ATTR_DESC(eps, AnyTraits())}}; +OUTPUT_MAP(L2NormalizeGrad) = {{0, OUTPUT_DESC(dx)}}; +REG_ADPT_DESC(L2NormalizeGrad, kNameL2NormalizeGrad, ADPT_DESC(L2NormalizeGrad)) + +// L2Normalize +INPUT_MAP(L2Normalize) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(L2Normalize) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}, + {"epsilon", ATTR_DESC(eps, AnyTraits())}}; +OUTPUT_MAP(L2Normalize) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(L2Normalize, kNameL2Normalize, ADPT_DESC(L2Normalize)) +} // namespace mindspore::transform -- 2.34.1 From 45160b679677706021dcb5ef0138b819bb127526 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:44:30 +0800 Subject: [PATCH 034/160] ADD file via upload --- .../graph_ir/nn_batch_norm_ops_declare.h | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.h new file mode 100644 index 00000000000..448eb3143c9 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_batch_norm_ops_declare.h @@ -0,0 +1,41 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_BATCH_NORM_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_BATCH_NORM_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/nn_batch_norm_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(BatchNorm)//将BatchNorm收录后与标准进行比较，进行空间调整，用input_map_为key储存 +DECLARE_OP_USE_OUTPUT(BatchNorm)//将BatchNorm收录后与标准进行比较，进行空间调整，用output_map_为key储存 + +DECLARE_OP_ADAPTER(BNInference) +DECLARE_OP_USE_OUTPUT(BNInference) + +DECLARE_OP_ADAPTER(BatchNormGrad) +DECLARE_OP_USE_OUTPUT(BatchNormGrad) + +DECLARE_OP_ADAPTER(L2Normalize) +DECLARE_OP_USE_OUTPUT(L2Normalize) + +DECLARE_OP_ADAPTER(L2NormalizeGrad) +DECLARE_OP_USE_OUTPUT(L2NormalizeGrad) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_BATCH_NORM_OPS_DECLARE_H_ -- 2.34.1 From bc0e81184120a432492bef310d0dc3e8b5e37673 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:44:46 +0800 Subject: [PATCH 035/160] ADD file via upload --- .../graph_ir/nn_calculation_ops_declare.cc | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.cc new file mode 100644 index 00000000000..2525a221d11 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.cc @@ -0,0 +1,239 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_calculation_ops_declare.h" +#include + +namespace mindspore::transform { +// BiasAddGrad +INPUT_MAP(BiasAddGrad) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(BiasAddGrad) = {{"format", ATTR_DESC(data_format, AnyTraits())}}; +OUTPUT_MAP(BiasAddGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BiasAddGrad, prim::kPrimBiasAddGrad->name(), ADPT_DESC(BiasAddGrad)) + +// Conv2D +INPUT_MAP(Conv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(Conv2D) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"group", ATTR_DESC(groups, AnyTraits())}, +}; +OUTPUT_MAP(Conv2D) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv2D, prim::kPrimConv2D->name(), ADPT_DESC(Conv2D)) + +// Conv2DBackpropInputD +//这一部分用到的宏定义比较全，我选择注释这一部分来概括一下 +INPUT_MAP(Conv2DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filter)}};/* +  将name变量out_backprop和filter处理并存入对应InputDesc结构体的相应变量中 +  将name变量内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为InputDesc结构体 +将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +*/ +INPUT_ATTR_MAP(Conv2DBackpropInputD) = { + {3, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}};/* +  将name变量input_size处理并存入对应AttrDesc结构体的相应变量中 +  将name变量内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为AttrDesc结构体 + +//其中AnyTraits<>的作用为将<>内类型进行构建 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 + +将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_attr_map_为key存储相应内容 +*/ +ATTR_MAP(Conv2DBackpropInputD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())},/* +  将name变量pads处理并存入对应AttrDesc结构体的相应变量中 +  将name变量内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为AttrDesc结构体 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 + +将收录内容的类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +*/ + {"stride", ATTR_DESC(strides, AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"group", ATTR_DESC(groups, AnyTraits())}, +}; +OUTPUT_MAP(Conv2DBackpropInputD) = {{0, OUTPUT_DESC(y)}};/* +  将name变量y处理并存入对应OutputDesc结构体的相应变量中 +  将name变量内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为OutputDesc结构体 + +将收录内容的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +*/ +REG_ADPT_DESC(Conv2DBackpropInputD, prim::kPrimConv2DBackpropInput->name(), ADPT_DESC(Conv2DBackpropInputD))/* + 将Conv2DBackpropInputD处理并存入对应ADPT_DESC结构体的相应变量中 +  将Conv2DBackpropInputD内容转为字符串变量并存储至结构体的name变量中 +  引用Operator空间并将指针所指的类转为ADPT_DESC结构体 + + 再将Conv2DBackpropInputD处理并存入对应REG_ADPT_DESC结构体的相应变量中 +  将Conv2DBackpropInputD内容转为字符串变量并存储至REG_ADPT_DESC的结构体的name变量中 +  引用Operator空间并将指针所指的类转为REG_ADPT_DESC结构体 +*/ + +// Conv2DBackpropInput for tf inference +INPUT_MAP(Conv2DBackpropInput) = {{1, INPUT_DESC(input_size)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}}; +ATTR_MAP(Conv2DBackpropInput) = { + {"stride", ATTR_DESC(strides, AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}, +}; +OUTPUT_MAP(Conv2DBackpropInput) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv2DBackpropInput, kNameConv2DBackpropInputV2, ADPT_DESC(Conv2DBackpropInput)) + +// Deconvolution for caffe inference +INPUT_MAP(Deconvolution) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +ATTR_MAP(Deconvolution) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"group", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"offset", ATTR_DESC(offset_x, AnyTraits())}}; +OUTPUT_MAP(Deconvolution) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Deconvolution, kNameDeconvolution, ADPT_DESC(Deconvolution)) +REG_ADPT_DESC(Conv2DTranspose, kConv2DTransposeOpName, ADPT_DESC(Conv2DBackpropInputD)) + +// Conv2DTransposeD for tf onnx inference +INPUT_MAP(Conv2DTransposeD) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +ATTR_MAP(Conv2DTransposeD) = { + {"input_size", ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"group", ATTR_DESC(groups, AnyTraits())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}, + {"output_paddings", ATTR_DESC(output_padding, AnyTraits>(), AnyTraits>())}, + {"offset", ATTR_DESC(offset_x, AnyTraits())}}; +OUTPUT_MAP(Conv2DTransposeD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv2DTransposeD, kNameConv2DTransposeD, ADPT_DESC(Conv2DTransposeD)) + +// Conv2DBackpropFilterD +INPUT_MAP(Conv2DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(Conv2DBackpropFilterD) = { + {3, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv2DBackpropFilterD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"group", ATTR_DESC(groups, AnyTraits())}, +}; +OUTPUT_MAP(Conv2DBackpropFilterD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv2DBackpropFilterD, prim::kPrimConv2DBackpropFilter->name(), ADPT_DESC(Conv2DBackpropFilterD)) + +// Conv3DTransposeD +INPUT_MAP(Conv3DTransposeD) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +ATTR_MAP(Conv3DTransposeD) = { + {"input_size", ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"output_padding", ATTR_DESC(output_padding, AnyTraits>(), AnyTraits>())}, +}; +OUTPUT_MAP(Conv3DTransposeD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv3DTransposeD, kNameConv3DTransposeD, ADPT_DESC(Conv3DTransposeD)) + +// Conv3D +INPUT_MAP(Conv3D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +ATTR_MAP(Conv3D) = { + {"strides", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"offset_x", ATTR_DESC(offset_x, AnyTraits())}, +}; +OUTPUT_MAP(Conv3D) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv3D, kNameConv3D, ADPT_DESC(Conv3D)) + +// Conv3DBackpropInputD +INPUT_MAP(Conv3DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filter)}}; +INPUT_ATTR_MAP(Conv3DBackpropInputD) = { + {3, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv3DBackpropInputD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, +}; +OUTPUT_MAP(Conv3DBackpropInputD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv3DBackpropInputD, kNameConv3DBackpropInputD, ADPT_DESC(Conv3DBackpropInputD)) + +// Conv3DBackpropFilterD +INPUT_MAP(Conv3DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(Conv3DBackpropFilterD) = { + {3, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv3DBackpropFilterD) = { + {"strides", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, +}; +OUTPUT_MAP(Conv3DBackpropFilterD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Conv3DBackpropFilterD, kNameConv3DBackpropFilterD, ADPT_DESC(Conv3DBackpropFilterD)) + +// DepthwiseConv2D +INPUT_MAP(DepthwiseConv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(DepthwiseConv2D) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, +}; +OUTPUT_MAP(DepthwiseConv2D) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(DepthwiseConv2D, prim::kPrimDepthwiseConv2dNative->name(), ADPT_DESC(DepthwiseConv2D)) + +// DepthwiseConv2DBackpropInputD +INPUT_MAP(DepthwiseConv2DBackpropInputD) = {{2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}}; +INPUT_ATTR_MAP(DepthwiseConv2DBackpropInputD) = { + {1, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(DepthwiseConv2DBackpropInputD) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, +}; +OUTPUT_MAP(DepthwiseConv2DBackpropInputD) = {{0, OUTPUT_DESC(input_grad)}}; +REG_ADPT_DESC(DepthwiseConv2DBackpropInputD, prim::kPrimDepthwiseConv2dNativeBackpropInput->name(), + ADPT_DESC(DepthwiseConv2DBackpropInputD)) + +// DepthwiseConv2DBackpropFilterD +INPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{1, INPUT_DESC(input)}, {3, INPUT_DESC(out_backprop)}}; +INPUT_ATTR_MAP(DepthwiseConv2DBackpropFilterD) = { + {2, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(DepthwiseConv2DBackpropFilterD) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, +}; +OUTPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{0, OUTPUT_DESC(filter_grad)}}; +REG_ADPT_DESC(DepthwiseConv2DBackpropFilterD, prim::kPrimDepthwiseConv2dNativeBackpropFilter->name(), + ADPT_DESC(DepthwiseConv2DBackpropFilterD)) +} // namespace mindspore::transform -- 2.34.1 From 211e16535f97d580bd9c4a1ef90c3a073c81f444 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:45:02 +0800 Subject: [PATCH 036/160] ADD file via upload --- .../graph_ir/nn_calculation_ops_declare.h | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.h new file mode 100644 index 00000000000..d2089f80edd --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_calculation_ops_declare.h @@ -0,0 +1,82 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_CALCULATION_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_CALCULATION_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/nn_calculation_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(BiasAddGrad)//函数功能为将BiasAddGrad的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(BiasAddGrad)//函数功能为将BiasAddGrad的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 + +DECLARE_OP_ADAPTER(Conv2D) +DECLARE_OP_USE_ENUM(Conv2D)//函数功能为将Conv2D的类型与标准进行对比，后进行空间调整，并用指针enum_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(Conv2D) + +DECLARE_OP_ADAPTER(Conv2DBackpropInputD) +DECLARE_OP_USE_ENUM(Conv2DBackpropInputD) +DECLARE_OP_USE_INPUT_ATTR(Conv2DBackpropInputD)////函数功能为将Conv2DBackpropInputD的类型与标准进行对比，后进行空间调整，并用指针input_attr_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(Conv2DBackpropInputD) + +DECLARE_OP_ADAPTER(Conv2DBackpropInput) +DECLARE_OP_USE_OUTPUT(Conv2DBackpropInput) + +DECLARE_OP_ADAPTER(Conv2DBackpropFilterD) +DECLARE_OP_USE_ENUM(Conv2DBackpropFilterD) +DECLARE_OP_USE_INPUT_ATTR(Conv2DBackpropFilterD) +DECLARE_OP_USE_OUTPUT(Conv2DBackpropFilterD) + +DECLARE_OP_ADAPTER(Conv3DTransposeD) +DECLARE_OP_USE_ENUM(Conv3DTransposeD) +DECLARE_OP_USE_OUTPUT(Conv3DTransposeD) + +DECLARE_OP_ADAPTER(Conv3D) +DECLARE_OP_USE_ENUM(Conv3D) +DECLARE_OP_USE_OUTPUT(Conv3D) + +DECLARE_OP_ADAPTER(Conv3DBackpropInputD) +DECLARE_OP_USE_ENUM(Conv3DBackpropInputD) +DECLARE_OP_USE_INPUT_ATTR(Conv3DBackpropInputD) +DECLARE_OP_USE_OUTPUT(Conv3DBackpropInputD) + +DECLARE_OP_ADAPTER(Conv3DBackpropFilterD) +DECLARE_OP_USE_ENUM(Conv3DBackpropFilterD) +DECLARE_OP_USE_INPUT_ATTR(Conv3DBackpropFilterD) +DECLARE_OP_USE_OUTPUT(Conv3DBackpropFilterD) + +DECLARE_OP_ADAPTER(DepthwiseConv2D) +DECLARE_OP_USE_ENUM(DepthwiseConv2D) +DECLARE_OP_USE_OUTPUT(DepthwiseConv2D) + +DECLARE_OP_ADAPTER(DepthwiseConv2DBackpropFilterD) +DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropFilterD) +DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropFilterD) + +DECLARE_OP_ADAPTER(DepthwiseConv2DBackpropInputD) +DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropInputD) +DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropInputD) + +DECLARE_OP_ADAPTER(Deconvolution) +DECLARE_OP_USE_OUTPUT(Deconvolution) + +DECLARE_OP_ADAPTER(Conv2DTransposeD) +DECLARE_OP_USE_OUTPUT(Conv2DTransposeD) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_CALCULATION_OPS_DECLARE_H_ -- 2.34.1 From 5a4490f631773277e9c2f1a74af09516b7b6191b Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:45:18 +0800 Subject: [PATCH 037/160] ADD file via upload --- .../graph_ir/nn_detect_ops_declare.cc | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.cc new file mode 100644 index 00000000000..91c86e03e44 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.cc @@ -0,0 +1,100 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_detect_ops_declare.h" +#include + +namespace mindspore::transform { +// BoundingBoxEncode +INPUT_MAP(BoundingBoxEncode) = { + {1, INPUT_DESC(anchor_box)}, + {2, INPUT_DESC(ground_truth_box)}, +};//将BoundingBoxEncode类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +ATTR_MAP(BoundingBoxEncode) = { + {"means", ATTR_DESC(means, AnyTraits>(), AnyTraits())}, + {"stds", ATTR_DESC(stds, AnyTraits>(), AnyTraits())}, +};/* +将means处理并存入对应 ATTR_DESC结构体的相应变量中 +将means内容转为字符串变量并存储至结构体的name变量中 +引用Operator空间并将指针所指的类转为 ATTR_DESC结构体 +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入MaxPool对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +*/ +//将BoundingBoxEncode类型与标准进行对比，后进行空间调整，并用指针attr_map_为key存储相应内容 +OUTPUT_MAP(BoundingBoxEncode) = {{0, OUTPUT_DESC(delats)}};//将delats处理并存入对应OUTPUT_DESC结构体的相应变量中 +  //将delats内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为OUTPUT_DESC结构体 + //将BoundingBoxEncode类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +REG_ADPT_DESC(BoundingBoxEncode, kNameBoundingBoxEncode, ADPT_DESC(BoundingBoxEncode))//将BoudingBoxEncode处理并存入对应RED_ADPT_DESC结构体的相应变量中 +  //将BoudingBoxEncode内容转为字符串变量并存储至结构体的name变量中 +  //引用Operator空间并将指针所指的类转为RED_ADPT_DESC结构体 + + +// BoundingBoxDecode +INPUT_MAP(BoundingBoxDecode) = { + {1, INPUT_DESC(rois)}, + {2, INPUT_DESC(deltas)}, +}; +ATTR_MAP(BoundingBoxDecode) = { + {"means", ATTR_DESC(means, AnyTraits>(), AnyTraits())}, + {"stds", ATTR_DESC(stds, AnyTraits>(), AnyTraits())}, + {"max_shape", ATTR_DESC(max_shape, AnyTraits>(), AnyTraits>())}, + {"wh_ratio_clip", ATTR_DESC(wh_ratio_clip, AnyTraits())}, +}; +OUTPUT_MAP(BoundingBoxDecode) = {{0, OUTPUT_DESC(bboxes)}}; +REG_ADPT_DESC(BoundingBoxDecode, kNameBoundingBoxDecode, ADPT_DESC(BoundingBoxDecode)) + +// Iou +INPUT_MAP(Iou) = {{1, INPUT_DESC(bboxes)}, {2, INPUT_DESC(gtboxes)}}; +ATTR_MAP(Iou) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +OUTPUT_MAP(Iou) = {{0, OUTPUT_DESC(overlap)}}; +REG_ADPT_DESC(Iou, kNameIOU, ADPT_DESC(Iou)) + +// CheckValid +INPUT_MAP(CheckValid) = {{1, INPUT_DESC(bbox_tensor)}, {2, INPUT_DESC(img_metas)}}; +ATTR_MAP(CheckValid) = EMPTY_ATTR_MAP; +OUTPUT_MAP(CheckValid) = {{0, OUTPUT_DESC(valid_tensor)}}; +REG_ADPT_DESC(CheckValid, kNameCheckValid, ADPT_DESC(CheckValid)) + +// Sort +INPUT_MAP(Sort) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Sort) = {{"axis", ATTR_DESC(axis, AnyTraits())}, + {"descending", ATTR_DESC(descending, AnyTraits())}}; +OUTPUT_MAP(Sort) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; +REG_ADPT_DESC(Sort, kNameSort, ADPT_DESC(Sort)) + +// ROIAlign +INPUT_MAP(ROIAlign) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(rois)}}; +OUTPUT_MAP(ROIAlign) = {{0, OUTPUT_DESC(y)}}; +ATTR_MAP(ROIAlign) = {{"pooled_height", ATTR_DESC(pooled_height, AnyTraits())}, + {"pooled_width", ATTR_DESC(pooled_width, AnyTraits())}, + {"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits())}, + {"sample_num", ATTR_DESC(sample_num, AnyTraits())}, + {"roi_end_mode", ATTR_DESC(roi_end_mode, AnyTraits())}}; +REG_ADPT_DESC(ROIAlign, kNameROIAlign, ADPT_DESC(ROIAlign)) +// ROIAlignGrad +INPUT_MAP(ROIAlignGrad) = {{1, INPUT_DESC(ydiff)}, {2, INPUT_DESC(rois)}}; +OUTPUT_MAP(ROIAlignGrad) = {{0, OUTPUT_DESC(xdiff)}}; +ATTR_MAP(ROIAlignGrad) = { + {"xdiff_shape", ATTR_DESC(xdiff_shape, AnyTraits>(), AnyTraits>())}, + {"pooled_height", ATTR_DESC(pooled_height, AnyTraits())}, + {"pooled_width", ATTR_DESC(pooled_width, AnyTraits())}, + {"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits())}, + {"sample_num", ATTR_DESC(sample_num, AnyTraits())}}; +REG_ADPT_DESC(ROIAlignGrad, kNameROIAlignGrad, ADPT_DESC(ROIAlignGrad)) +} // namespace mindspore::transform -- 2.34.1 From 9760639ec1c5df748fe0e80dd6fceb5bc9b58325 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:45:38 +0800 Subject: [PATCH 038/160] ADD file via upload --- .../graph_ir/nn_detect_ops_declare.h | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.h new file mode 100644 index 00000000000..2df72ac0808 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_detect_ops_declare.h @@ -0,0 +1,47 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_DETECT_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_DETECT_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/nn_detect_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(Iou)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 +DECLARE_OP_USE_OUTPUT(Iou)//将收录内容的类型与标准进行对比，后进行空间调整，并用指针output_map_为key存储相应内容 +//以下部分根据变量的不同对不同变量进行操作 +DECLARE_OP_ADAPTER(CheckValid) +DECLARE_OP_USE_OUTPUT(CheckValid) + +DECLARE_OP_ADAPTER(Sort) +DECLARE_OP_USE_OUTPUT(Sort) + +DECLARE_OP_ADAPTER(BoundingBoxEncode) +DECLARE_OP_USE_OUTPUT(BoundingBoxEncode) + +DECLARE_OP_ADAPTER(BoundingBoxDecode) +DECLARE_OP_USE_OUTPUT(BoundingBoxDecode) + +DECLARE_OP_ADAPTER(ROIAlign) +DECLARE_OP_USE_OUTPUT(ROIAlign) + +DECLARE_OP_ADAPTER(ROIAlignGrad) +DECLARE_OP_USE_OUTPUT(ROIAlignGrad) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_DETECT_OPS_DECLARE_H_ -- 2.34.1 From 6b88ba9edeed632aa14200ba86272ae1bafa418b Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:46:02 +0800 Subject: [PATCH 039/160] ADD file via upload --- .../transform/graph_ir/nn_norm_ops_declare.cc | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.cc new file mode 100644 index 00000000000..d8ae448520d --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.cc @@ -0,0 +1,163 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_norm_ops_declare.h" +#include + +namespace mindspore::transform { +// SoftmaxV2 +INPUT_MAP(SoftmaxV2) = {{1, INPUT_DESC(x)}};//将SoftmaxV2与标准进行比较，以input_map_为key并构造名为x的INPUT_DESC结构体并与标准比较其容量 +ATTR_MAP(SoftmaxV2) = { + {"axis", ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}, +};//将SoftmaxV2与标准进行比较，以attr_map_为key并构造储存了name变量axes的ATTR_DESC结构体 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(SoftmaxV2) = {{0, OUTPUT_DESC(y)}};//将SoftmaxV2与标准进行比较，以output_map_为key，并构造储存了name变量y的OUTPUT_DESC结构体并与标准比较其容量 +REG_ADPT_DESC(SoftmaxV2, kNameSoftmax, ADPT_DESC(SoftmaxV2))//在ADPT_DESC结构体的基础上创建名为kNameSoftmax的RED_ADPT_DESC结构体 + //将SoftmaxV2转为字符变量存入结构体的name变量中 + //并与标准比较其容量，返回对应内容 +//以下部分根据引用的不同变量对其进行相应操作 +// SoftmaxGrad +INPUT_MAP(SoftmaxGrad) = {{1, INPUT_DESC(softmax)}, {2, INPUT_DESC(grad_softmax)}}; +OUTPUT_MAP(SoftmaxGrad) = {{0, OUTPUT_DESC(grad_x)}}; +ATTR_MAP(SoftmaxGrad) = EMPTY_ATTR_MAP; +REG_ADPT_DESC(SoftmaxGrad, kNameSoftmaxGrad, ADPT_DESC(SoftmaxGrad)) + +// SoftmaxCrossEntropyWithLogits +INPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(labels)}}; +ATTR_MAP(SoftmaxCrossEntropyWithLogits) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(backprop)}}; +REG_ADPT_DESC(SoftmaxCrossEntropyWithLogits, prim::kPrimSoftmaxCrossEntropyWithLogits->name(), + ADPT_DESC(SoftmaxCrossEntropyWithLogits)) + +// SmoothL1Loss +INPUT_MAP(SmoothL1Loss) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}}; +ATTR_MAP(SmoothL1Loss) = {{"beta", ATTR_DESC(sigma, AnyTraits())}}; +OUTPUT_MAP(SmoothL1Loss) = {{0, OUTPUT_DESC(loss)}}; +REG_ADPT_DESC(SmoothL1Loss, kNameSmoothL1Loss, ADPT_DESC(SmoothL1Loss)) + +// SmoothL1LossGrad +INPUT_MAP(SmoothL1LossGrad) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}, {3, INPUT_DESC(dout)}}; +ATTR_MAP(SmoothL1LossGrad) = {{"beta", ATTR_DESC(sigma, AnyTraits())}}; +OUTPUT_MAP(SmoothL1LossGrad) = {{0, OUTPUT_DESC(gradient)}}; +REG_ADPT_DESC(SmoothL1LossGrad, kNameSmoothL1LossGrad, ADPT_DESC(SmoothL1LossGrad)) + +// SigmoidCrossEntropyWithLogits +INPUT_MAP(SigmoidCrossEntropyWithLogits) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}}; +ATTR_MAP(SigmoidCrossEntropyWithLogits) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SigmoidCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}}; +REG_ADPT_DESC(SigmoidCrossEntropyWithLogits, kNameSigmoidCrossEntropyWithLogits, + ADPT_DESC(SigmoidCrossEntropyWithLogits)) + +// SigmoidCrossEntropyWithLogitsGrad +INPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = { + {1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(dout)}}; +ATTR_MAP(SigmoidCrossEntropyWithLogitsGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = {{0, OUTPUT_DESC(gradient)}}; +REG_ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad, kNameSigmoidCrossEntropyWithLogitsGrad, + ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad)) + +// SigmoidCrossEntropyWithLogitsV2 +INPUT_MAP(SigmoidCrossEntropyWithLogitsV2) = { + {1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(weight)}, {4, INPUT_DESC(pos_weight)}}; +ATTR_MAP(SigmoidCrossEntropyWithLogitsV2) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(SigmoidCrossEntropyWithLogitsV2) = {{0, OUTPUT_DESC(loss)}}; +REG_ADPT_DESC(SigmoidCrossEntropyWithLogitsV2, kNameSigmoidCrossEntropyWithLogitsV2, + ADPT_DESC(SigmoidCrossEntropyWithLogitsV2)) + +// LogSoftmaxGrad +INPUT_MAP(LogSoftmaxGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}}; +ATTR_MAP(LogSoftmaxGrad) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +OUTPUT_MAP(LogSoftmaxGrad) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(LogSoftmaxGrad, prim::kPrimLogSoftmaxGrad->name(), ADPT_DESC(LogSoftmaxGrad)) + +// LogSoftmaxV2 +INPUT_MAP(LogSoftmaxV2) = {{1, INPUT_DESC(logits)}}; +ATTR_MAP(LogSoftmaxV2) = { + {"axis", ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +OUTPUT_MAP(LogSoftmaxV2) = {{0, OUTPUT_DESC(logsoftmax)}}; +REG_ADPT_DESC(LogSoftmaxV2, prim::kPrimLogSoftmax->name(), ADPT_DESC(LogSoftmaxV2)) + +// LayerNorm +INPUT_MAP(LayerNorm) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(gamma)}, {3, INPUT_DESC(beta)}}; +ATTR_MAP(LayerNorm) = {{"begin_norm_axis", ATTR_DESC(begin_norm_axis, AnyTraits())}, + {"begin_params_axis", ATTR_DESC(begin_params_axis, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}};//将begin_norm_axis变量处理并存入结构体对应的变量中，转为字符串并存至name变量中， + //收录begin_+norm_axis变量和结构体，并与标准进行比较，进行空间调整并用attr_map_为key储存相应内容 +OUTPUT_MAP(LayerNorm) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mean)}, {2, OUTPUT_DESC(variance)}}; +REG_ADPT_DESC(LayerNorm, prim::kPrimLayerNorm->name(), ADPT_DESC(LayerNorm)) + +// LayerNormGrad +INPUT_MAP(LayerNormGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(dy)}, {3, INPUT_DESC(variance)}, {4, INPUT_DESC(mean)}, {5, INPUT_DESC(gamma)}}; +ATTR_MAP(LayerNormGrad) = EMPTY_ATTR_MAP; +OUTPUT_MAP(LayerNormGrad) = {{0, OUTPUT_DESC(pd_x)}, {1, OUTPUT_DESC(pd_gamma)}, {2, OUTPUT_DESC(pd_beta)}}; +REG_ADPT_DESC(LayerNormGrad, prim::kPrimLayerNormGrad->name(), ADPT_DESC(LayerNormGrad)) + +// LRN +INPUT_MAP(LRN) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(LRN) = {{"depth_radius", ATTR_DESC(depth_radius, AnyTraits())}, + {"bias", ATTR_DESC(bias, AnyTraits())}, + {"alpha", ATTR_DESC(alpha, AnyTraits())}, + {"beta", ATTR_DESC(beta, AnyTraits())}, + {"norm_region", ATTR_DESC(norm_region, AnyTraits())}}; +OUTPUT_MAP(LRN) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(LRN, kNameLRN, ADPT_DESC(LRN)) + +// LRNGrad +INPUT_MAP(LRNGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}}; +ATTR_MAP(LRNGrad) = {{"depth_radius", ATTR_DESC(depth_radius, AnyTraits())}, + {"bias", ATTR_DESC(bias, AnyTraits())}, + {"alpha", ATTR_DESC(alpha, AnyTraits())}, + {"beta", ATTR_DESC(beta, AnyTraits())}}; +OUTPUT_MAP(LRNGrad) = {{0, OUTPUT_DESC(z)}}; +REG_ADPT_DESC(LRNGrad, kNameLRNGrad, ADPT_DESC(LRNGrad)) + +// DropoutDoMask +INPUT_MAP(DropOutDoMask) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mask)}, {3, INPUT_DESC(keep_prob)}}; +ATTR_MAP(DropOutDoMask) = EMPTY_ATTR_MAP; +OUTPUT_MAP(DropOutDoMask) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(DropOutDoMask, kNameDropoutDoMask, ADPT_DESC(DropOutDoMask)) + +// BinaryCrossEntropy +INPUT_MAP(BinaryCrossEntropy) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(weight)}}; +ATTR_MAP(BinaryCrossEntropy) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(BinaryCrossEntropy) = {{0, OUTPUT_DESC(output)}}; +REG_ADPT_DESC(BinaryCrossEntropy, kNameBinaryCrossEntropy, ADPT_DESC(BinaryCrossEntropy)) + +// BinaryCrossEntropyGrad +INPUT_MAP(BinaryCrossEntropyGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(grad_output)}, {4, INPUT_DESC(weight)}}; +ATTR_MAP(BinaryCrossEntropyGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +OUTPUT_MAP(BinaryCrossEntropyGrad) = {{0, OUTPUT_DESC(output)}}; +REG_ADPT_DESC(BinaryCrossEntropyGrad, kNameBinaryCrossEntropyGrad, ADPT_DESC(BinaryCrossEntropyGrad)) + +// Centralization +INPUT_MAP(Centralization) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Centralization) = {{"axes", ATTR_DESC(axes, AnyTraits>())}}; +OUTPUT_MAP(Centralization) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Centralization, kNameCentralization, ADPT_DESC(Centralization)) + +// Scale +INPUT_MAP(Scale) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(scale)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(Scale) = {{"axis", ATTR_DESC(axis, AnyTraits())}, + {"num_axes", ATTR_DESC(num_axes, AnyTraits())}, + {"scale_from_blob", ATTR_DESC(scale_from_blob, AnyTraits())}}; + +OUTPUT_MAP(Scale) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(Scale, kNameScale, ADPT_DESC(Scale)) +} // namespace mindspore::transform -- 2.34.1 From 89003127bf0047f2130f96a34ddd2e06842dc6b9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:46:25 +0800 Subject: [PATCH 040/160] ADD file via upload --- .../transform/graph_ir/nn_norm_ops_declare.h | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.h new file mode 100644 index 00000000000..23bea22e03e --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_norm_ops_declare.h @@ -0,0 +1,83 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_ + +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/nn_norm_ops.h" + +namespace mindspore::transform { +DECLARE_OP_ADAPTER(SmoothL1Loss) +DECLARE_OP_USE_OUTPUT(SmoothL1Loss) + +DECLARE_OP_ADAPTER(SmoothL1LossGrad) +DECLARE_OP_USE_OUTPUT(SmoothL1LossGrad) + +DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogits) +DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogits) + +DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogitsGrad) +DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogitsGrad) + +DECLARE_OP_ADAPTER(SigmoidCrossEntropyWithLogitsV2) +DECLARE_OP_USE_OUTPUT(SigmoidCrossEntropyWithLogitsV2) + +DECLARE_OP_ADAPTER(LogSoftmaxGrad) +DECLARE_OP_USE_OUTPUT(LogSoftmaxGrad) + +DECLARE_OP_ADAPTER(LogSoftmaxV2) +DECLARE_OP_USE_OUTPUT(LogSoftmaxV2) + +DECLARE_OP_ADAPTER(LayerNorm) +DECLARE_OP_USE_OUTPUT(LayerNorm) + +DECLARE_OP_ADAPTER(LayerNormGrad) +DECLARE_OP_USE_OUTPUT(LayerNormGrad) + +DECLARE_OP_ADAPTER(LRN) +DECLARE_OP_USE_OUTPUT(LRN) + +DECLARE_OP_ADAPTER(LRNGrad) +DECLARE_OP_USE_OUTPUT(LRNGrad) + +DECLARE_OP_ADAPTER(DropOutDoMask) +DECLARE_OP_USE_OUTPUT(DropOutDoMask) + +DECLARE_OP_ADAPTER(SoftmaxCrossEntropyWithLogits) +DECLARE_OP_USE_OUTPUT(SoftmaxCrossEntropyWithLogits) + +DECLARE_OP_ADAPTER(SoftmaxV2) +DECLARE_OP_USE_OUTPUT(SoftmaxV2) + +DECLARE_OP_ADAPTER(SoftmaxGrad) +DECLARE_OP_USE_OUTPUT(SoftmaxGrad) + +DECLARE_OP_ADAPTER(BinaryCrossEntropy) +DECLARE_OP_USE_OUTPUT(BinaryCrossEntropy) + +DECLARE_OP_ADAPTER(BinaryCrossEntropyGrad) +DECLARE_OP_USE_OUTPUT(BinaryCrossEntropyGrad) + +DECLARE_OP_ADAPTER(Centralization) +DECLARE_OP_USE_OUTPUT(Centralization) + +DECLARE_OP_ADAPTER(Scale) +DECLARE_OP_USE_OUTPUT(Scale) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_IMAGE_OPS_DECLARE_H_ -- 2.34.1 From c575c5d5164691419fa9ab71ed4e606168315ae1 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:46:48 +0800 Subject: [PATCH 041/160] ADD file via upload --- .../graph_ir/nn_pooling_ops_declare.cc | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.cc new file mode 100644 index 00000000000..3d08e7bb294 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.cc @@ -0,0 +1,232 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_pooling_ops_declare.h"//按照路径寻找以下文件，导入到本文件 +#include //提供vector数组构建函数模版等 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// MaxPool最大池化 +INPUT_MAP(MaxPool) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPool对应空间内并用input_map指针保存 +ATTR_MAP(MaxPool) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPool对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPool) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPool, kNameMaxPool, ADPT_DESC(MaxPool))//构造指向MaxPool的指针并储存，创建结构体RegAdptDescMaxPool + +// MaxPool3D +INPUT_MAP(MaxPool3D) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPool3D对应空间内并用input_map指针保存 +ATTR_MAP(MaxPool3D) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"pad_list", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilation, AnyTraits(), AnyTraits>())}, + {"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPool3D对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPool3D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool3D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPool3D, kNameMaxPool3D, ADPT_DESC(MaxPool3D))//构造指向MaxPool3D的指针并储存，创建结构体RegAdptDescMaxPool3D + +// MaxPool3DGrad +INPUT_MAP(MaxPool3DGrad) = {{1, INPUT_DESC(orig_x)}, {2, INPUT_DESC(orig_y)}, {3, INPUT_DESC(grads)}}; +//将变量orig_x、orig_y、grads处理并存入对应InputDesc结构体的相应变量中，存入MaxPool3DGrad对应空间内并用input_map指针保存 +ATTR_MAP(MaxPool3DGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPool3DGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPool3DGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool3DGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPool3DGrad, kNameMaxPool3DGrad, ADPT_DESC(MaxPool3DGrad))//构造指向MaxPool3DGrad的指针并储存，创建结构体RegAdptDescMaxPool3DGrad + +// MaxPool3DGradGrad +INPUT_MAP(MaxPool3DGradGrad) = {{1, INPUT_DESC(orig_x)}, {2, INPUT_DESC(orig_y)}, {3, INPUT_DESC(grads)}}; +//将变量orig_x、orig_y、grads处理并存入对应InputDesc结构体的相应变量中，存入MaxPool3DGradGrad对应空间内并用input_map指针保存 +ATTR_MAP(MaxPool3DGradGrad) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPool3DGradGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPool3DGradGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPool3DGradGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPool3DGradGrad, kNameMaxPool3DGradGrad, ADPT_DESC(MaxPool3DGradGrad))//构造指向MaxPool3DGrad的指针并储存，创建结构体RegAdptDescMaxPool3DGradGrad + +// AvgPool平均池化 +INPUT_MAP(AvgPool) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入AvgPool对应空间内并用input_map指针保存 +ATTR_MAP(AvgPool) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入AvgPool对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(AvgPool) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AvgPool对应空间内并用output_map_指针保存 +REG_ADPT_DESC(AvgPool, kNameAvgPool, ADPT_DESC(AvgPool))//构造指向AvgPool的指针并储存，创建结构体RegAdptDescAvgPool + +// MaxPoolGrad +INPUT_MAP(MaxPoolGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}}; +//将变量x1、x2、x3处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGrad对应空间内并用input_map指针保存 +ATTR_MAP(MaxPoolGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPoolGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPoolGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPoolGrad, kNameMaxPoolGrad, ADPT_DESC(MaxPoolGrad))//构造指向MaxPoolGrad的指针并储存，创建结构体RegAdptDescMaxPoolGrad + +// MaxPoolGradGrad +INPUT_MAP(MaxPoolGradGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}}; +//将变量x1、x2、x3处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGraddGrad对应空间内并用input_map指针保存 +ATTR_MAP(MaxPoolGradGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPoolGradGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPoolGradGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGradGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPoolGradGrad, kNameMaxPoolGradGrad, ADPT_DESC(MaxPoolGradGrad))//构造指向MaxPoolGradGrad的指针并储存，创建结构体RegAdptDescMaxPoolGradGrad + +// avgpoolgrad +INPUT_MAP(AvgPoolGrad) = {{1, INPUT_DESC(orig_input_shape)}, {2, INPUT_DESC(input_grad)}}; +//将变量orig_input_shape、input_grad处理并存入对应InputDesc结构体的相应变量中，存入avgpoolgrad对应空间内并用input_map指针保存 +ATTR_MAP(AvgPoolGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入AvgPoolGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(AvgPoolGrad) = {{0, OUTPUT_DESC(out_grad)}};//将变量out_grad处理并存入对应OutputDesc结构体的相应变量中，存入AvgPoolGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(AvgPoolGrad, kNameAvgPoolGrad, ADPT_DESC(AvgPoolGrad))//构造指向AvgPoolGrad的指针并储存，创建结构体RegAdptDescAvgPoolGrad + +// MaxPoolWithArgmax +INPUT_MAP(MaxPoolWithArgmax) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolWithArgmax对应空间内并用input_map指针保存 +ATTR_MAP(MaxPoolWithArgmax) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPoolWithArgmax对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPoolWithArgmax) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(argmax)}}; +//将变量y、argmax处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolWithArgmax对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPoolWithArgmax, kNameMaxPoolWithArgmax, ADPT_DESC(MaxPoolWithArgmax)) +//构造指向MaxPoolWithArgmax的指针并储存，创建结构体RegAdptDescMaxPoolWithArgmax + +// MaxPoolGradWithArgmax +INPUT_MAP(MaxPoolGradWithArgmax) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}, {3, INPUT_DESC(argmax)}}; +//将变量x、grad、argmax处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGradWithArgmax对应空间内并用input_map指针保存 +ATTR_MAP(MaxPoolGradWithArgmax) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPoolGradWithArgmax对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPoolGradWithArgmax) = {{0, OUTPUT_DESC(y)}}; +//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGradWithArgmax对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPoolGradWithArgmax, kNameMaxPoolGradWithArgmax, ADPT_DESC(MaxPoolGradWithArgmax)) +//构造指向MaxPoolGradWithArgmax的指针并储存，创建结构体RegAdptDescMaxPoolGradWithArgmax + +// MaxPoolGradGradWithArgmax +INPUT_MAP(MaxPoolGradGradWithArgmax) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}, {3, INPUT_DESC(argmax)}}; +//将变量x、grad、argmax处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolGradGradWithArgmax对应空间内并用input_map指针保存 +ATTR_MAP(MaxPoolGradGradWithArgmax) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPoolGradGradWithArgmax对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPoolGradGradWithArgmax) = {{0, OUTPUT_DESC(y)}}; +//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolGradGradWithArgmax对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPoolGradGradWithArgmax, kNameMaxPoolGradGradWithArgmax, ADPT_DESC(MaxPoolGradGradWithArgmax)) +//构造指向MaxPoolGradGradWithArgmax的指针并储存，创建结构体RegAdptDescMaxPoolGradGradWithArgmax + +// Pooling +INPUT_MAP(Pooling) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Pooling对应空间内并用input_map指针保存 +ATTR_MAP(Pooling) = {{"mode", ATTR_DESC(mode, AnyTraits())}, + {"global", ATTR_DESC(global_pooling, AnyTraits())}, + {"kernel_size", ATTR_DESC(window, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(stride, AnyTraits(), AnyTraits>())}, + {"pad", ATTR_DESC(pad, AnyTraits(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilation, AnyTraits(), AnyTraits>())}, + {"round_mode", ATTR_DESC(ceil_mode, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入Pooling对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(Pooling) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Pooling对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Pooling, kNamePooling, ADPT_DESC(Pooling))//构造指向Pooling的指针并储存，创建结构体RegAdptDescPooling + +// MaxPoolV3 +INPUT_MAP(MaxPoolV3) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入MaxPoolV3对应空间内并用input_map指针保存 +ATTR_MAP(MaxPoolV3) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding_mode", ATTR_DESC(padding_mode, AnyTraits())}, + {"pad", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"global", ATTR_DESC(global_pooling, AnyTraits())}, + {"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入MaxPoolV3对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(MaxPoolV3) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入MaxPoolV3对应空间内并用output_map_指针保存 +REG_ADPT_DESC(MaxPoolV3, kNameMaxPoolV3, ADPT_DESC(MaxPoolV3))//构造指向MaxPoolV3的指针并储存，创建结构体RegAdptDescMaxPoolV3 + +// AvgPoolV2 +INPUT_MAP(AvgPoolV2) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入AvgPoolV2对应空间内并用input_map指针保存 +ATTR_MAP(AvgPoolV2) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding_mode", ATTR_DESC(padding_mode, AnyTraits())}, + {"pad", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"global", ATTR_DESC(global_pooling, AnyTraits())}, + {"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入AvgPoolV2对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(AvgPoolV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AvgPoolV2对应空间内并用output_map_指针保存 +REG_ADPT_DESC(AvgPoolV2, kNameAvgPoolV2, ADPT_DESC(AvgPoolV2))//构造指向AvgPoolV2的指针并储存，创建结构体RegAdptDescAvgPoolV2 + +// GlobalAveragePool +INPUT_MAP(GlobalAveragePool) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入GlobalAveragePool对应空间内并用input_map指针保存 +ATTR_MAP(GlobalAveragePool) = EMPTY_ATTR_MAP;//将空变量存入AvgPoolV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(GlobalAveragePool) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GlobalAveragePool对应空间内并用output_map_指针保存 +REG_ADPT_DESC(GlobalAveragePool, kNameGlobalAvgPool, ADPT_DESC(GlobalAveragePool))//构造指向GlobalAveragePool的指针并储存，创建结构体RegAdptDescGlobalAveragePool + +// Upsample +INPUT_MAP(Upsample) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Upsample对应空间内并用input_map指针保存 +ATTR_MAP(Upsample) = {{"scale", ATTR_DESC(scale, AnyTraits())}, + {"stride_h", ATTR_DESC(stride_h, AnyTraits())}, + {"stride_w", ATTR_DESC(stride_w, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入Upsample对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(Upsample) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Upsample对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Upsample, kNameUpsample, ADPT_DESC(Upsample))//构造指向Upsample的指针并储存，创建结构体RegAdptDescUpsample +} // namespace mindspore::transform -- 2.34.1 From e58b2e820c39f704af6ac2ec9b1ab95b9b9958c2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:47:11 +0800 Subject: [PATCH 042/160] ADD file via upload --- .../graph_ir/nn_pooling_ops_declare.h | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.h new file mode 100644 index 00000000000..b06ef5f952f --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_pooling_ops_declare.h @@ -0,0 +1,76 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/nn_ops.h" +#include "ops/nn_pooling_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现MaxPoolWithArgmax的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPoolWithArgmax) +DECLARE_OP_USE_OUTPUT(MaxPoolWithArgmax) +//对实现MaxPoolGradWithArgmax的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPoolGradWithArgmax) +DECLARE_OP_USE_OUTPUT(MaxPoolGradWithArgmax) +//对实现MaxPoolGradGradWithArgmax的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPoolGradGradWithArgmax) +DECLARE_OP_USE_OUTPUT(MaxPoolGradGradWithArgmax) +//对实现MaxPool的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPool) +DECLARE_OP_USE_OUTPUT(MaxPool) +//对实现MaxPoolGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPoolGrad) +DECLARE_OP_USE_OUTPUT(MaxPoolGrad) +//对实现MaxPoolGradGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPoolGradGrad) +DECLARE_OP_USE_OUTPUT(MaxPoolGradGrad) +//对实现MaxPool3D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPool3D) +DECLARE_OP_USE_OUTPUT(MaxPool3D) +//对实现MaxPool3DGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPool3DGrad) +DECLARE_OP_USE_OUTPUT(MaxPool3DGrad) +//对实现MaxPool3DGradGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPool3DGradGrad) +DECLARE_OP_USE_OUTPUT(MaxPool3DGradGrad) +//对实现AvgPool的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(AvgPool) +DECLARE_OP_USE_OUTPUT(AvgPool) +//对实现AvgPoolGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(AvgPoolGrad) +DECLARE_OP_USE_OUTPUT(AvgPoolGrad) +//对实现Pooling的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Pooling) +DECLARE_OP_USE_OUTPUT(Pooling) +//对实现MaxPoolV3的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(MaxPoolV3) +DECLARE_OP_USE_OUTPUT(MaxPoolV3) +//对实现AvgPoolV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(AvgPoolV2) +DECLARE_OP_USE_OUTPUT(AvgPoolV2) +//对实现GlobalAveragePool的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(GlobalAveragePool) +DECLARE_OP_USE_OUTPUT(GlobalAveragePool) +//对实现Upsample的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Upsample) +DECLARE_OP_USE_OUTPUT(Upsample) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_POOLING_OPS_DECLARE_H_ -- 2.34.1 From a9ecad999c76414fedc6e8f0ff780be158de1146 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:47:29 +0800 Subject: [PATCH 043/160] ADD file via upload --- .../graph_ir/nn_training_ops_declare.cc | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.cc new file mode 100644 index 00000000000..749521252a7 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.cc @@ -0,0 +1,319 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_training_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// ApplyMomentum +INPUT_MAP(ApplyMomentum) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}, {5, INPUT_DESC(momentum)}}; +//将变量var、accum、lr、grad、momentum处理并存入对应InputDesc结构体的相应变量中，存入ApplyMomentum对应空间内并用input_map指针保存 +ATTR_MAP(ApplyMomentum) = {{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyMomentum对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyMomentum) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyMomentum对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyMomentum, kNameApplyMomentum, ADPT_DESC(ApplyMomentum)) +//构造指向ApplyMomentum的指针并储存，创建结构体RegAdptDescApplyMomentum + +// LarsV2Update +INPUT_MAP(LarsV2Update) = {{1, INPUT_DESC(w)}, + {2, INPUT_DESC(g)}, + {3, INPUT_DESC(w_square_sum)}, + {4, INPUT_DESC(g_square_sum)}, + {5, INPUT_DESC(weight_decay)}, + {6, INPUT_DESC(learning_rate)}}; +//将变量w、g、w_square_sum、g_square_sum、weight_decay、learning_rate处理并存入对应InputDesc结构体的相应变量中，存入arsV2Update对应空间内并用input_map指针保存 +ATTR_MAP(LarsV2Update) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"hyperpara", ATTR_DESC(hyperpara, AnyTraits())}, + {"use_clip", ATTR_DESC(use_clip, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入LarsV2Update对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(LarsV2Update) = {{0, OUTPUT_DESC(g_new)}};//将变量g_new处理并存入对应OutputDesc结构体的相应变量中，存入LarsV2Update对应空间内并用output_map_指针保存 +REG_ADPT_DESC(LarsV2Update, kNameLARSUpdate, ADPT_DESC(LarsV2Update))//构造指向LarsV2Update的指针并储存，创建结构体RegAdptDescLarsV2Update + +// ApplyAdam +INPUT_MAP(ApplyAdam) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)}, + {10, INPUT_DESC(grad)}}; +//将变量var、m、v、beta1_power、beta2_power、lr、beta1、beta2、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdam对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAdam) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAdam对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAdam) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdam对应空间内并用output_map_指针保存 + +// ApplyAdamD +INPUT_MAP(ApplyAdamD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)}, + {10, INPUT_DESC(grad)}}; +//将变量var、m、v、beta1_power、beta2_power、lr、beta1、beta2、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdamD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAdamD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAdamD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAdamD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}}; +//将变量var、m、v处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdamD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyAdamD, kNameApplyAdam, ADPT_DESC(ApplyAdamD))//构造指向ApplyAdamD的指针并储存，创建结构体RegAdptDescApplyAdamD +REG_ADPT_DESC(ApplyAdam, kNameApplyAdam, ADPT_DESC(ApplyAdam))//构造指向ApplyAdam的指针并储存，创建结构体RegAdptDescApplyAdam + +// ApplyAdagradD +INPUT_MAP(ApplyAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}}; +//将变量var、accum、lr、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdagradD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAdagradD) = {{"update_slots", ATTR_DESC(update_slots, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAdagradD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +//将变量var、accum处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdagradD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyAdagradD, kNameApplyAdagrad, ADPT_DESC(ApplyAdagradD))//构造指向ApplyAdagradD的指针并储存，创建结构体RegAdptDescApplyAdagradD + +// ApplyAdagradV2D +INPUT_MAP(ApplyAdagradV2D) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}}; +//将变量var、accum、lr、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdagradV2D对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAdagradV2D) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"update_slots", ATTR_DESC(update_slots, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAdagradV2D对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAdagradV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +//将变量var、accum处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdagradV2D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyAdagradV2D, kNameApplyAdagradV2D, ADPT_DESC(ApplyAdagradV2D))//构造指向ApplyAdagradV2D的指针并储存，创建结构体RegAdptDescApplyAdagradV2D + +// ApplyAddSignD +INPUT_MAP(ApplyAddSignD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(alpha)}, {5, INPUT_DESC(sign_decay)}, {6, INPUT_DESC(beta)}, + {7, INPUT_DESC(grad)}}; +//将变量var、m、lr、alpha、sign_decay、beta、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAddSignD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAddSignD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAddSignD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAddSignD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}}; +//将变量var、m处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAddSignD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyAddSignD, kNameApplyAddSignD, ADPT_DESC(ApplyAddSignD))//构造指向ApplyAddSignD的指针并储存，创建结构体RegAdptDescApplyAddSignD + +// SparseApplyAdagradV2D +INPUT_MAP(SparseApplyAdagradV2D) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}}; +//将变量var、accum、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyAdagradV2D对应空间内并用input_map指针保存 +ATTR_MAP(SparseApplyAdagradV2D) = {{"lr", ATTR_DESC(lr, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"update_slots", ATTR_DESC(update_slots, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SparseApplyAdagradV2D对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SparseApplyAdagradV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +//将变量var、accum处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyAdagradV2D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SparseApplyAdagradV2D, kNameSparseApplyAdagradV2D, ADPT_DESC(SparseApplyAdagradV2D)) +//构造指向SparseApplyAdagradV2D的指针并储存，创建结构体RegAdptDescSparseApplyAdagradV2D + +// DataFormatDimMap +INPUT_MAP(DataFormatDimMap) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入DataFormatDimMap对应空间内并用input_map指针保存 +ATTR_MAP(DataFormatDimMap) = {{"src_format", ATTR_DESC(src_format, AnyTraits())}, + {"dst_format", ATTR_DESC(dst_format, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DataFormatDimMap对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DataFormatDimMap) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入DataFormatDimMap对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DataFormatDimMap, kNameDataFormatDimMap, ADPT_DESC(DataFormatDimMap)) +//构造指向DataFormatDimMap的指针并储存，创建结构体RegAdptDescDataFormatDimMap + +// ApplyAdadeltaD +INPUT_MAP(ApplyAdadeltaD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(accum_update)}, + {4, INPUT_DESC(lr)}, {5, INPUT_DESC(rho)}, {6, INPUT_DESC(epsilon)}, + {7, INPUT_DESC(grad)}}; +//将变量var、accum、accum_update、lr、rho、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdadeltaD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAdadeltaD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAdadeltaD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAdadeltaD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}, {2, OUTPUT_DESC(accum_update)}}; +//将变量var、accum、accum_update处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdadeltaD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyAdadeltaD, kNameApplyAdadelta, ADPT_DESC(ApplyAdadeltaD)) +//构造指向ApplyAdadeltaD的指针并储存，创建结构体RegAdptDescApplyAdadeltaD + +// ApplyAdaMaxD +INPUT_MAP(ApplyAdaMaxD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(beta1)}, + {7, INPUT_DESC(beta2)}, {8, INPUT_DESC(epsilon)}, {9, INPUT_DESC(grad)}}; +//将变量var、m、v、beta1_power、lr、beta1、beta2、epsilon、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyAdaMaxD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyAdaMaxD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyAdaMaxD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyAdaMaxD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}}; +//将变量var、m、v处理并存入对应OutputDesc结构体的相应变量中，存入ApplyAdaMaxD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyAdaMaxD, kNameApplyAdaMax, ADPT_DESC(ApplyAdaMaxD)) +//构造指向ApplyAdaMaxD的指针并储存，创建结构体RegAdptDescApplyAdaMaxD + +// ApplyGradientDescent +INPUT_MAP(ApplyGradientDescent) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(alpha)}, {3, INPUT_DESC(delta)}}; +//将变量var、alpha、delta处理并存入对应InputDesc结构体的相应变量中，存入ApplyGradientDescent对应空间内并用input_map指针保存 +ATTR_MAP(ApplyGradientDescent) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyGradientDescent对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyGradientDescent) = {{0, OUTPUT_DESC(var)}}; +//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyGradientDescent对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyGradientDescent, kNameApplyGradientDescent, ADPT_DESC(ApplyGradientDescent)) +//构造指向ApplyGradientDescent的指针并储存，创建结构体RegAdptDescApplyGradientDescent + +// ApplyPowerSignD +INPUT_MAP(ApplyPowerSignD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(logbase)}, {5, INPUT_DESC(sign_decay)}, {6, INPUT_DESC(beta)}, + {7, INPUT_DESC(grad)}}; +//将变量var、m、lr、logbase、sign_decay、beta、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyPowerSignD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyPowerSignD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyPowerSignD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyGradientDescent) = {{0, OUTPUT_DESC(var)}}; +OUTPUT_MAP(ApplyPowerSignD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}}; +//将变量var、m处理并存入对应OutputDesc结构体的相应变量中，存入ApplyPowerSignDt对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyPowerSignD, kNameApplyPowerSign, ADPT_DESC(ApplyPowerSignD)) +//构造指向ApplyPowerSignD的指针并储存，创建结构体RegAdptDescApplyPowerSignD + +// ApplyProximalGradientDescent +INPUT_MAP(ApplyProximalGradientDescent) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(alpha)}, {3, INPUT_DESC(l1)}, {4, INPUT_DESC(l2)}, {5, INPUT_DESC(delta)}}; +//将变量var、alpha、l1、l2、delta处理并存入对应InputDesc结构体的相应变量中，存入ApplyProximalGradientDescent对应空间内并用input_map指针保存 +ATTR_MAP(ApplyProximalGradientDescent) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyProximalGradientDescent对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyProximalGradientDescent) = {{0, OUTPUT_DESC(var)}}; +//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyProximalGradientDescent对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyProximalGradientDescent, kNameApplyProximalGradientDescent, ADPT_DESC(ApplyProximalGradientDescent)) +//构造指向ApplyProximalGradientDescent的指针并储存，创建结构体RegAdptDescApplyProximalGradientDescent + +// SGD +INPUT_MAP(SGD) = {{1, INPUT_DESC(parameters)}, {2, INPUT_DESC(gradient)}, {3, INPUT_DESC(learning_rate)}, + {4, INPUT_DESC(accum)}, {5, INPUT_DESC(momentum)}, {6, INPUT_DESC(stat)}}; +//将变量parameters、gradient、learning_rate、accum、momentum、stat处理并存入对应InputDesc结构体的相应变量中，存入SGD对应空间内并用input_map指针保存 +ATTR_MAP(SGD) = {{"dampening", ATTR_DESC(dampening, AnyTraits())}, + {"weight_decay", ATTR_DESC(weight_decay, AnyTraits())}, + {"nesterov", ATTR_DESC(nesterov, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SGD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SGD) = {{0, OUTPUT_DESC(parameters)}};//将变量parameters处理并存入对应OutputDesc结构体的相应变量中，存入SGD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SGD, kNameSGD, ADPT_DESC(SGD))//构造指向SGD的指针并储存，创建结构体RegAdptDescSGD + +// SparseApplyAdagradD +INPUT_MAP(SparseApplyAdagradD) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}}; +//将变量var、accum、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyAdagradD对应空间内并用input_map指针保存 +ATTR_MAP(SparseApplyAdagradD) = {{"lr", ATTR_DESC(lr, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SparseApplyAdagradD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SparseApplyAdagradD) = {{0, OUTPUT_DESC(var)}}; +//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyAdagradD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SparseApplyAdagradD, kNameSparseApplyAdagrad, ADPT_DESC(SparseApplyAdagradD)) +//构造指向SparseApplyAdagradD的指针并储存，创建结构体RegAdptDescSparseApplyAdagradD + +// ApplyProximalAdagradD +INPUT_MAP(ApplyProximalAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(l1)}, {5, INPUT_DESC(l2)}, {6, INPUT_DESC(grad)}}; +//将变量var、accum、lr、l1、l2、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyProximalAdagradD对应空间内并用input_map指针保存 +ATTR_MAP(ApplyProximalAdagradD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyProximalAdagradD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyProximalAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +//将变量accum处理并存入对应OutputDesc结构体的相应变量中，存入ApplyProximalAdagradD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyProximalAdagradD, kNameApplyProximalAdagrad, ADPT_DESC(ApplyProximalAdagradD)) +//构造指向ApplyProximalAdagradD的指针并储存，创建结构体RegAdptDescApplyProximalAdagradD + +// SparseApplyProximalAdagradD +INPUT_MAP(SparseApplyProximalAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(l1)}, {5, INPUT_DESC(l2)}, {6, INPUT_DESC(grad)}, + {7, INPUT_DESC(indices)}}; +//将变量var、accum、lr、l1、l2、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyProximalAdagradD对应空间内并用input_map指针保存 +ATTR_MAP(SparseApplyProximalAdagradD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SparseApplyProximalAdagradD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SparseApplyProximalAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +//将变量accum处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyProximalAdagradD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SparseApplyProximalAdagradD, kNameSparseApplyProximalAdagradD, ADPT_DESC(SparseApplyProximalAdagradD)) +//构造指向SparseApplyProximalAdagradD的指针并储存，创建结构体RegAdptDescSparseApplyProximalAdagradD + +// SparseApplyFtrlD +INPUT_MAP(SparseApplyFtrlD) = {{1, INPUT_DESC(var)}, + {2, INPUT_DESC(accum)}, + {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, + {5, INPUT_DESC(indices)}}; +//将变量var、accum、linear、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyFtrlD对应空间内并用input_map指针保存 +ATTR_MAP(SparseApplyFtrlD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"lr", ATTR_DESC(lr, AnyTraits())}, + {"l1", ATTR_DESC(l1, AnyTraits())}, + {"l2", ATTR_DESC(l2, AnyTraits())}, + {"lr_power", ATTR_DESC(lr_power, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SparseApplyFtrlD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SparseApplyFtrlD) = {{0, OUTPUT_DESC(var)}}; +//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyFtrlD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SparseApplyFtrlD, kNameSparseApplyFtrlD, ADPT_DESC(SparseApplyFtrlD)) +//构造指向SparseApplyFtrlD的指针并储存，创建结构体RegAdptDescSparseApplyFtrlD + +// SparseApplyFtrlV2D +INPUT_MAP(SparseApplyFtrlV2D) = {{1, INPUT_DESC(var)}, + {2, INPUT_DESC(accum)}, + {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, + {5, INPUT_DESC(indices)}}; +//将变量var、accum、linear、grad、indices处理并存入对应InputDesc结构体的相应变量中，存入SparseApplyFtrlV2D对应空间内并用input_map指针保存 +ATTR_MAP(SparseApplyFtrlV2D) = {{"lr", ATTR_DESC(lr, AnyTraits())}, {"l1", ATTR_DESC(l1, AnyTraits())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入SparseApplyFtrlV2D对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SparseApplyFtrlV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}, {2, OUTPUT_DESC(linear)}}; +//将变量var、accum、linear处理并存入对应OutputDesc结构体的相应变量中，存入SparseApplyFtrlV2D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SparseApplyFtrlV2D, kNameSparseApplyFtrlV2D, ADPT_DESC(SparseApplyFtrlV2D)) +//构造指向SparseApplyFtrlV2D的指针并储存，创建结构体RegAdptDescSparseApplyFtrlV2D + +// ApplyFtrl +INPUT_MAP(ApplyFtrl) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(l1)}, + {7, INPUT_DESC(l2)}, {8, INPUT_DESC(lr_power)}}; +//将变量var、accum、linear、grad、lr、l1、l2、lr_power处理并存入对应InputDesc结构体的相应变量中，存入ApplyFtrl对应空间内并用input_map指针保存 +ATTR_MAP(ApplyFtrl) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyFtrl对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyFtrl) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyFtrl对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyFtrl, kNameApplyFtrl, ADPT_DESC(ApplyFtrl))//构造指向ApplyFtrl的指针并储存，创建结构体RegAdptDescApplyFtrl + +// ApplyRMSPropD +INPUT_MAP(ApplyRMSPropD) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(ms)}, {3, INPUT_DESC(mom)}, {4, INPUT_DESC(lr)}, {5, INPUT_DESC(grad)}}; +//将变量var、ms、mom、lr、grad处理并存入对应InputDesc结构体的相应变量中，存入ApplyRMSPropD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ApplyRMSPropD) = {{6, ATTR_DESC(rho, AnyTraits())}, + {7, ATTR_DESC(momentum, AnyTraits())}, + {8, ATTR_DESC(epsilon, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyRMSPropD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(ApplyRMSPropD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ApplyRMSPropD) = {{0, OUTPUT_DESC(var)}};//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyRMSPropD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyRMSPropD, kNameApplyRMSProp, ADPT_DESC(ApplyRMSPropD))//构造指向ApplyRMSPropD的指针并储存，创建结构体RegAdptDescApplyRMSPropD + +// ApplyCenteredRMSProp +INPUT_MAP(ApplyCenteredRMSProp) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(mg)}, {3, INPUT_DESC(ms)}, + {4, INPUT_DESC(mom)}, {5, INPUT_DESC(grad)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(rho)}, {8, INPUT_DESC(momentum)}, {9, INPUT_DESC(epsilon)}}; +//将变量var、mg、ms、mom、grad、lr、rho、momentum、epsilon处理并存入对应InputDesc结构体的相应变量中，存入ApplyCenteredRMSProp对应空间内并用input_map指针保存 +ATTR_MAP(ApplyCenteredRMSProp) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ApplyCenteredRMSProp对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ApplyCenteredRMSProp) = {{0, OUTPUT_DESC(var)}}; +//将变量var处理并存入对应OutputDesc结构体的相应变量中，存入ApplyCenteredRMSProp对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ApplyCenteredRMSProp, kNameApplyCenteredRMSProp, ADPT_DESC(ApplyCenteredRMSProp)) +//构造指向ApplyCenteredRMSProp的指针并储存，创建结构体RegAdptDescApplyCenteredRMSProp +} // namespace mindspore::transform -- 2.34.1 From a954b885f60292af15f5111011e7b4b7ec6bf518 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:49:24 +0800 Subject: [PATCH 044/160] ADD file via upload --- .../graph_ir/nn_training_ops_declare.h | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.h new file mode 100644 index 00000000000..5987a778b35 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nn_training_ops_declare.h @@ -0,0 +1,97 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/nn_training_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现ApplyAdam的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAdam) +DECLARE_OP_USE_OUTPUT(ApplyAdam) +//对实现ApplyAdamD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAdamD) +DECLARE_OP_USE_OUTPUT(ApplyAdamD) +//对实现ApplyAdagradD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAdagradD) +DECLARE_OP_USE_OUTPUT(ApplyAdagradD) +//对实现ApplyAdagradV2D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAdagradV2D) +DECLARE_OP_USE_OUTPUT(ApplyAdagradV2D) +//对实现ApplyAddSignD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAddSignD) +DECLARE_OP_USE_OUTPUT(ApplyAddSignD) +//对实现SparseApplyAdagradV2D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SparseApplyAdagradV2D) +DECLARE_OP_USE_OUTPUT(SparseApplyAdagradV2D) +//对实现DataFormatDimMap的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DataFormatDimMap) +DECLARE_OP_USE_OUTPUT(DataFormatDimMap) +//对实现ApplyAdadeltaD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAdadeltaD) +DECLARE_OP_USE_OUTPUT(ApplyAdadeltaD) +//对实现ApplyAdaMaxD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyAdaMaxD) +DECLARE_OP_USE_OUTPUT(ApplyAdaMaxD) +//对实现ApplyGradientDescent的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyGradientDescent) +DECLARE_OP_USE_OUTPUT(ApplyGradientDescent) +//对实现ApplyPowerSignD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyPowerSignD) +DECLARE_OP_USE_OUTPUT(ApplyPowerSignD) +//对实现ApplyProximalGradientDescent的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyProximalGradientDescent) +DECLARE_OP_USE_OUTPUT(ApplyProximalGradientDescent) +//对实现SGD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SGD) +DECLARE_OP_USE_OUTPUT(SGD) +//对实现ApplyMomentum的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyMomentum) +DECLARE_OP_USE_OUTPUT(ApplyMomentum) +//对实现SparseApplyAdagradD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SparseApplyAdagradD) +DECLARE_OP_USE_OUTPUT(SparseApplyAdagradD) +//对实现ApplyProximalAdagradD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyProximalAdagradD) +DECLARE_OP_USE_OUTPUT(ApplyProximalAdagradD) +//对实现SparseApplyProximalAdagradD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SparseApplyProximalAdagradD) +DECLARE_OP_USE_OUTPUT(SparseApplyProximalAdagradD) +//对实现LarsV2Update的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(LarsV2Update) +DECLARE_OP_USE_OUTPUT(LarsV2Update) +//对实现ApplyFtrl的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyFtrl) +DECLARE_OP_USE_OUTPUT(ApplyFtrl) +//对实现SparseApplyFtrlD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SparseApplyFtrlD) +DECLARE_OP_USE_OUTPUT(SparseApplyFtrlD) +//对实现SparseApplyFtrlV2D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SparseApplyFtrlV2D) +DECLARE_OP_USE_OUTPUT(SparseApplyFtrlV2D) +//对实现ApplyRMSPropD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyRMSPropD) +DECLARE_OP_USE_INPUT_ATTR(ApplyRMSPropD) +DECLARE_OP_USE_OUTPUT(ApplyRMSPropD) +//对实现ApplyCenteredRMSProp的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ApplyCenteredRMSProp) +DECLARE_OP_USE_OUTPUT(ApplyCenteredRMSProp) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NN_TRAINING_OPS_DECLARE_H_ -- 2.34.1 From 1ad771c314642ca3d0aa48590b03fb736c1c4b55 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:49:42 +0800 Subject: [PATCH 045/160] ADD file via upload --- .../graph_ir/nonlinear_fuc_ops_declare.cc | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.cc new file mode 100644 index 00000000000..668789e252f --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.cc @@ -0,0 +1,204 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nonlinear_fuc_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// Relu +INPUT_MAP(Relu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Relu对应空间内并用input_map指针保存 +ATTR_MAP(Relu) = EMPTY_ATTR_MAP;//将空变量存入Relu对应空间并用attr_map_指针保存 +OUTPUT_MAP(Relu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Relu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Relu, prim::kPrimRelu->name(), ADPT_DESC(Relu))//构造指向Relu的指针并储存，创建结构体RegAdptDescRelu + +// ReluV2 +INPUT_MAP(ReluV2) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReluV2对应空间内并用input_map指针保存 +ATTR_MAP(ReluV2) = EMPTY_ATTR_MAP;//将空变量存入ReluV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(ReluV2) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}}; +//将变量y、mask处理并存入对应OutputDesc结构体的相应变量中，存入ReluV2对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReluV2, kNameReluV2, ADPT_DESC(ReluV2))//构造指向ReluV2的指针并储存，创建结构体RegAdptDescReluV2 + +// Elu +INPUT_MAP(Elu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Elu对应空间内并用input_map指针保存 +ATTR_MAP(Elu) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入Elu对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(Elu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Elu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Elu, kNameElu, ADPT_DESC(Elu))//构造指向Elu的指针并储存，创建结构体RegAdptDescElu + +// EluGrad +INPUT_MAP(EluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(activations)}}; +//将变量grads、activations处理并存入对应InputDesc结构体的相应变量中，存入EluGrad对应空间内并用input_map指针保存 +ATTR_MAP(EluGrad) = EMPTY_ATTR_MAP;//将空变量存入EluGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(EluGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入EluGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(EluGrad, kNameEluGrad, ADPT_DESC(EluGrad))//构造指向EluGrad的指针并储存，创建结构体RegAdptDescEluGrad + +// PRelu +INPUT_MAP(PRelu) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(weight)}}; +//将变量x、weight处理并存入对应InputDesc结构体的相应变量中，存入PRelu对应空间内并用input_map指针保存 +ATTR_MAP(PRelu) = EMPTY_ATTR_MAP;//将空变量存入PRelu对应空间并用attr_map_指针保存 +OUTPUT_MAP(PRelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PRelu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(PRelu, kNamePrelu, ADPT_DESC(PRelu))//构造指向PRelu的指针并储存，创建结构体RegAdptDescPRelu + +// PReluGrad +INPUT_MAP(PReluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(features)}, {3, INPUT_DESC(weights)}}; +//将变量grads、features、weights处理并存入对应InputDesc结构体的相应变量中，存入PReluGrad对应空间内并用input_map指针保存 +ATTR_MAP(PReluGrad) = EMPTY_ATTR_MAP;//将空变量存入PReluGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(PReluGrad) = {{0, OUTPUT_DESC(dx)}, {1, OUTPUT_DESC(da)}}; +//将变量dx、da处理并存入对应OutputDesc结构体的相应变量中，存入PReluGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(PReluGrad, kNamePreluGrad, ADPT_DESC(PReluGrad))//构造指向PReluGrad的指针并储存，创建结构体RegAdptDescPReluGrad + +// Selu +INPUT_MAP(Selu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Selu对应空间内并用input_map指针保存 +ATTR_MAP(Selu) = EMPTY_ATTR_MAP;//将空变量存入Selu对应空间并用attr_map_指针保存 +OUTPUT_MAP(Selu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Selu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Selu, kNameSelu, ADPT_DESC(Selu))//构造指向Selu的指针并储存，创建结构体RegAdptDescSelu + +// Sigmoid +INPUT_MAP(Sigmoid) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Sigmoid对应空间内并用input_map指针保存 +ATTR_MAP(Sigmoid) = EMPTY_ATTR_MAP;//将空变量存入Sigmoid对应空间并用attr_map_指针保存 +OUTPUT_MAP(Sigmoid) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Sigmoid对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Sigmoid, kNameSigmoid, ADPT_DESC(Sigmoid))//构造指向Sigmoid的指针并储存，创建结构体RegAdptDescSigmoid + +// SigmoidGrad +INPUT_MAP(SigmoidGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +//将变量y、dy处理并存入对应InputDesc结构体的相应变量中，存入SigmoidGrad对应空间内并用input_map指针保存 +ATTR_MAP(SigmoidGrad) = EMPTY_ATTR_MAP;//将空变量存入SigmoidGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(SigmoidGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入SigmoidGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SigmoidGrad, kNameSigmoidGrad, ADPT_DESC(SigmoidGrad))//构造指向SigmoidGrad的指针并储存，创建结构体RegAdptDescSigmoidGrad + +// HardSwish +INPUT_MAP(HardSwish) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入HardSwish对应空间内并用input_map指针保存 +ATTR_MAP(HardSwish) = EMPTY_ATTR_MAP;//将空变量存入HardSwish对应空间并用attr_map_指针保存 +OUTPUT_MAP(HardSwish) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入HardSwish对应空间内并用output_map_指针保存 +REG_ADPT_DESC(HardSwish, kNameHSwish, ADPT_DESC(HardSwish))//构造指向HardSwish的指针并储存，创建结构体RegAdptDescHardSwish + +// HardSwishGrad +INPUT_MAP(HardSwishGrad) = {{1, INPUT_DESC(grad)}, {2, INPUT_DESC(x)}}; +//将变量grad、x处理并存入对应InputDesc结构体的相应变量中，存入HardSwishGrad对应空间内并用input_map指针保存 +ATTR_MAP(HardSwishGrad) = EMPTY_ATTR_MAP;//将空变量存入HardSwishGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(HardSwishGrad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入HardSwishGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(HardSwishGrad, kNameHSwishGrad, ADPT_DESC(HardSwishGrad))//构造指向HardSwishGrad的指针并储存，创建结构体RegAdptDescHardSwishGrad + +// HSigmoid +INPUT_MAP(HardSigmoid) = {{1, INPUT_DESC(input_x)}};//将变量input_x处理并存入对应InputDesc结构体的相应变量中，存入HardSigmoid对应空间内并用input_map指针保存 +ATTR_MAP(HardSigmoid) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}, + {"beta", ATTR_DESC(beta, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入HardSigmoid对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(HardSigmoid) = {{0, OUTPUT_DESC(output_y)}};//将变量output_y处理并存入对应OutputDesc结构体的相应变量中，存入HardSigmoid对应空间内并用output_map_指针保存 +REG_ADPT_DESC(HardSigmoid, kNameHSigmoid, ADPT_DESC(HardSigmoid))//构造指向HardSigmoid的指针并储存，创建结构体RegAdptDescHardSigmoid + +// Relu6 +INPUT_MAP(Relu6) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Relu6对应空间内并用input_map指针保存 +ATTR_MAP(Relu6) = EMPTY_ATTR_MAP;//将空变量存入Relu6对应空间并用attr_map_指针保存 +OUTPUT_MAP(Relu6) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Relu6对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Relu6, kNameReLU6, ADPT_DESC(Relu6))//构造指向Relu6的指针并储存，创建结构体RegAdptDescRelu6 + +// Relu6Grad +INPUT_MAP(Relu6Grad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +//将变量gradients、features处理并存入对应InputDesc结构体的相应变量中，存入Relu6Grad对应空间内并用input_map指针保存 +ATTR_MAP(Relu6Grad) = EMPTY_ATTR_MAP;//将空变量存入Relu6Grad对应空间并用attr_map_指针保存 +OUTPUT_MAP(Relu6Grad) = {{0, OUTPUT_DESC(backprops)}};//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入Relu6Grad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Relu6Grad, kNameReLU6Grad, ADPT_DESC(Relu6Grad))//构造指向Relu6Grad的指针并储存，创建结构体RegAdptDescRelu6Grad + +// Softsign +INPUT_MAP(Softsign) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Softsign对应空间内并用input_map指针保存 +ATTR_MAP(Softsign) = EMPTY_ATTR_MAP;//将空变量存入Softsign对应空间并用attr_map_指针保存 +OUTPUT_MAP(Softsign) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Softsign对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Softsign, kNameSoftsign, ADPT_DESC(Softsign))//构造指向Softsign的指针并储存，创建结构体RegAdptDescSoftsign + +// Softplus +INPUT_MAP(Softplus) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Softplus对应空间内并用input_map指针保存 +ATTR_MAP(Softplus) = EMPTY_ATTR_MAP;//将空变量存入Softplus对应空间并用attr_map_指针保存 +OUTPUT_MAP(Softplus) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Softplus对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Softplus, kNameSoftplus, ADPT_DESC(Softplus))//构造指向Softplus的指针并储存，创建结构体RegAdptDescSoftplus + +// SoftplusGrad +INPUT_MAP(SoftplusGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +//将变量gradients、features处理并存入对应InputDesc结构体的相应变量中，存入SoftplusGrad对应空间内并用input_map指针保存 +ATTR_MAP(SoftplusGrad) = EMPTY_ATTR_MAP;//将空变量存入SoftplusGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(SoftplusGrad) = {{0, OUTPUT_DESC(backprops)}}; +//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入SoftplusGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SoftplusGrad, kNameSoftplusGrad, ADPT_DESC(SoftplusGrad))//构造指向SoftplusGrad的指针并储存，创建结构体RegAdptDescSoftplusGrad + +// ReluGrad +INPUT_MAP(ReluGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +//将变量gradients、features处理并存入对应InputDesc结构体的相应变量中，存入ReluGrad对应空间内并用input_map指针保存 +ATTR_MAP(ReluGrad) = EMPTY_ATTR_MAP;//将空变量存入ReluGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(ReluGrad) = {{0, OUTPUT_DESC(backprops)}};//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入ReluGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReluGrad, prim::kPrimReluGrad->name(), ADPT_DESC(ReluGrad))//构造指向ReluGrad的指针并储存，创建结构体RegAdptDescReluGrad + +// ReluGradV2 +INPUT_MAP(ReluGradV2) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(mask)}}; +//将变量gradients、mask处理并存入对应InputDesc结构体的相应变量中，存入ReluGradV2对应空间内并用input_map指针保存 +ATTR_MAP(ReluGradV2) = EMPTY_ATTR_MAP;//将空变量存入ReluGradV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(ReluGradV2) = {{0, OUTPUT_DESC(backprops)}};//将变量backprops处理并存入对应OutputDesc结构体的相应变量中，存入ReluGradV2对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReluGradV2, kNameReluGradV2, ADPT_DESC(ReluGradV2))//构造指向ReluGradV2的指针并储存，创建结构体RegAdptDescReluGradV2 + +// Tanh +INPUT_MAP(Tanh) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Tanh对应空间内并用input_map指针保存 +ATTR_MAP(Tanh) = EMPTY_ATTR_MAP;//将空变量存入ReluGradV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(Tanh) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Tanh对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Tanh, prim::kPrimTanh->name(), ADPT_DESC(Tanh))//构造指向Tanh的指针并储存，创建结构体RegAdptDescTanh + +// TanhGrad +INPUT_MAP(TanhGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +//将变量y、dy处理并存入对应InputDesc结构体的相应变量中，存入TanhGrad对应空间内并用input_map指针保存 +ATTR_MAP(TanhGrad) = EMPTY_ATTR_MAP;//将空变量存入ReluGradV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(TanhGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入TanhGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(TanhGrad, prim::kPrimTanhGrad->name(), ADPT_DESC(TanhGrad))//构造指向TanhGrad的指针并储存，创建结构体RegAdptDescTanhGrad + +// Mish +INPUT_MAP(Mish) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Mish对应空间内并用input_map指针保存 +ATTR_MAP(Mish) = EMPTY_ATTR_MAP;//将空变量存入Mish对应空间并用attr_map_指针保存 +OUTPUT_MAP(Mish) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Mish对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Mish, kNameMish, ADPT_DESC(Mish))//构造指向Mish的指针并储存，创建结构体RegAdptDescMish + +// GeLU +INPUT_MAP(Gelu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Gelu对应空间内并用input_map指针保存 +ATTR_MAP(Gelu) = EMPTY_ATTR_MAP;//将空变量存入Gelu对应空间并用attr_map_指针保存 +OUTPUT_MAP(Gelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Gelu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Gelu, prim::kPrimGeLU->name(), ADPT_DESC(Gelu))//构造指向Gelu的指针并储存，创建结构体RegAdptDescGelu + +// GeLUGrad +INPUT_MAP(GeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}}; +//将变量dy、x、y处理并存入对应InputDesc结构体的相应变量中，存入GeluGrad对应空间内并用input_map指针保存 +ATTR_MAP(GeluGrad) = EMPTY_ATTR_MAP;//将空变量存入GeluGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(GeluGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入GeluGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(GeluGrad, prim::kPrimGeLUGrad->name(), ADPT_DESC(GeluGrad))//构造指向GeluGrad的指针并储存，创建结构体RegAdptDescGeluGrad + +// FastGeLU +INPUT_MAP(FastGelu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入FastGelu对应空间内并用input_map指针保存 +ATTR_MAP(FastGelu) = EMPTY_ATTR_MAP;//将空变量存入FastGelu对应空间并用attr_map_指针保存 +OUTPUT_MAP(FastGelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入FastGelu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(FastGelu, prim::kPrimFastGeLU->name(), ADPT_DESC(FastGelu))//构造指向FastGelu的指针并储存，创建结构体RegAdptDescFastGelu + +// FastGeLUGrad +INPUT_MAP(FastGeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}}; +//将变量dy、x处理并存入对应InputDesc结构体的相应变量中，存入FastGeluGrad对应空间内并用input_map指针保存 +ATTR_MAP(FastGeluGrad) = EMPTY_ATTR_MAP;//将空变量存入FastGeluGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(FastGeluGrad) = {{0, OUTPUT_DESC(z)}};//将变量z处理并存入对应OutputDesc结构体的相应变量中，存入FastGeluGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(FastGeluGrad, prim::kPrimFastGeLUGrad->name(), ADPT_DESC(FastGeluGrad)) +//构造指向FastGeluGrad的指针并储存，创建结构体RegAdptDescFastGeluGrad + +// LeakyRelu +INPUT_MAP(LeakyRelu) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入LeakyRelu对应空间内并用input_map指针保存 +ATTR_MAP(LeakyRelu) = {{"alpha", ATTR_DESC(negative_slope, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入LeakyRelu对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(LeakyRelu) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入LeakyRelu对应空间内并用output_map_指针保存 +REG_ADPT_DESC(LeakyRelu, prim::kPrimLeakyRelu->name(), ADPT_DESC(LeakyRelu))//构造指向LeakyRelu的指针并储存，创建结构体RegAdptDescLeakyRelu +} // namespace mindspore::transform -- 2.34.1 From fd73216b77731a752968ec8318a2cffb693284ff Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:49:59 +0800 Subject: [PATCH 046/160] ADD file via upload --- .../graph_ir/nonlinear_fuc_ops_declare.h | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.h new file mode 100644 index 00000000000..572b6a69cc4 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/nonlinear_fuc_ops_declare.h @@ -0,0 +1,108 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "ops/nonlinear_fuc_ops.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现ReluGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReluGrad) +DECLARE_OP_USE_OUTPUT(ReluGrad) +//对实现ReluGradV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReluGradV2) +DECLARE_OP_USE_OUTPUT(ReluGradV2) +//对实现Relu6的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Relu6) +DECLARE_OP_USE_OUTPUT(Relu6) +//对实现Relu6Grad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Relu6Grad) +DECLARE_OP_USE_OUTPUT(Relu6Grad) +//对实现Softsign的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Softsign) +DECLARE_OP_USE_OUTPUT(Softsign) +//对实现Softplus的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Softplus) +DECLARE_OP_USE_OUTPUT(Softplus) +//对实现SoftplusGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SoftplusGrad) +DECLARE_OP_USE_OUTPUT(SoftplusGrad) +//对实现Tanh的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Tanh) +DECLARE_OP_USE_OUTPUT(Tanh) +//对实现TanhGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(TanhGrad) +DECLARE_OP_USE_OUTPUT(TanhGrad) +//对实现Mish的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Mish) +DECLARE_OP_USE_OUTPUT(Mish) +//对实现Gelu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Gelu) +DECLARE_OP_USE_OUTPUT(Gelu) +//对实现GeluGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(GeluGrad) +DECLARE_OP_USE_OUTPUT(GeluGrad) +//对实现FastGelu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(FastGelu) +DECLARE_OP_USE_OUTPUT(FastGelu) +//对实现FastGeluGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(FastGeluGrad) +DECLARE_OP_USE_OUTPUT(FastGeluGrad) +//对实现Relu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Relu) +DECLARE_OP_USE_OUTPUT(Relu) +//对实现ReluV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReluV2) +DECLARE_OP_USE_OUTPUT(ReluV2) +//对实现PRelu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(PRelu) +DECLARE_OP_USE_OUTPUT(PRelu) +//对实现Elu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Elu) +DECLARE_OP_USE_OUTPUT(Elu) +//对实现EluGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(EluGrad) +DECLARE_OP_USE_OUTPUT(EluGrad) +//对实现PReluGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(PReluGrad) +DECLARE_OP_USE_OUTPUT(PReluGrad) +//对实现Selu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Selu) +DECLARE_OP_USE_OUTPUT(Selu) +//对实现Sigmoid的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Sigmoid) +DECLARE_OP_USE_OUTPUT(Sigmoid) +//对实现HardSwish的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(HardSwish) +DECLARE_OP_USE_OUTPUT(HardSwish) +//对实现HardSwishGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(HardSwishGrad) +DECLARE_OP_USE_OUTPUT(HardSwishGrad) +//对实现HardSigmoid的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(HardSigmoid) +DECLARE_OP_USE_OUTPUT(HardSigmoid) +//对实现SigmoidGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SigmoidGrad) +DECLARE_OP_USE_OUTPUT(SigmoidGrad) +//对实现LeakyRelu的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(LeakyRelu) +DECLARE_OP_USE_OUTPUT(LeakyRelu) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NONLINEAR_FUC_OPS_DECLARE_H_ -- 2.34.1 From 52ae9908cd315276c473118fabc58e0d333fa08a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:50:15 +0800 Subject: [PATCH 047/160] ADD file via upload --- .../graph_ir/npu_loss_scale_ops_declare.cc | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.cc new file mode 100644 index 00000000000..4b218036f17 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/npu_loss_scale_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// NPUGetFloatStatus +INPUT_MAP(NPUGetFloatStatus) = {{1, INPUT_DESC(addr)}};//将变量addr处理并存入对应InputDesc结构体的相应变量中，存入NPUGetFloatStatus对应空间内并用input_map指针保存 +OUTPUT_MAP(NPUGetFloatStatus) = {{0, OUTPUT_DESC(data)}}; +//将变量data处理并存入对应OutputDesc结构体的相应变量中，存入NPUGetFloatStatus对应空间内并用output_map_指针保存 +ATTR_MAP(NPUGetFloatStatus) = EMPTY_ATTR_MAP;//将空变量存入NPUGetFloatStatus对应空间并用attr_map_指针保存 +REG_ADPT_DESC(NPUGetFloatStatus, kNameNPUGetFloatStatus, ADPT_DESC(NPUGetFloatStatus)) +//构造指向NPUGetFloatStatus的指针并储存，创建结构体RegAdptDescNPUGetFloatStatus + +// NPUAllocFloatStatus +INPUT_MAP(NPUAllocFloatStatus) = EMPTY_INPUT_MAP;//将空变量存入NPUAllocFloatStatus对应空间内并用input_map指针保存 +ATTR_MAP(NPUAllocFloatStatus) = EMPTY_ATTR_MAP;//将空变量存入NPUAllocFloatStatus对应空间并用attr_map_指针保存 +OUTPUT_MAP(NPUAllocFloatStatus) = {{0, OUTPUT_DESC(data)}}; +//将变量data处理并存入对应OutputDesc结构体的相应变量中，存入NPUAllocFloatStatus对应空间内并用output_map_指针保存 +REG_ADPT_DESC(NPUAllocFloatStatus, kNameNPUAllocFloatStatus, ADPT_DESC(NPUAllocFloatStatus)) +//构造指向NPUAllocFloatStatus的指针并储存，创建结构体RegAdptDescNPUAllocFloatStatus + +// NPUClearFloatStatus +INPUT_MAP(NPUClearFloatStatus) = {{1, INPUT_DESC(addr)}};//将变量addr处理并存入对应InputDesc结构体的相应变量中，存入NPUClearFloatStatus对应空间内并用input_map指针保存 +OUTPUT_MAP(NPUClearFloatStatus) = {{0, OUTPUT_DESC(data)}}; +//将变量data处理并存入对应OutputDesc结构体的相应变量中，存入NPUClearFloatStatus对应空间内并用output_map_指针保存 +ATTR_MAP(NPUClearFloatStatus) = EMPTY_ATTR_MAP;//将空变量存入NPUClearFloatStatus对应空间并用attr_map_指针保存 +REG_ADPT_DESC(NPUClearFloatStatus, kNameNPUClearFloatStatus, ADPT_DESC(NPUClearFloatStatus)) +//构造指向NPUClearFloatStatus的指针并储存，创建结构体RegAdptDescNPUClearFloatStatus +} // namespace mindspore::transform -- 2.34.1 From 75cde5d8d057932d0475deda56c7e84b777c1609 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:50:46 +0800 Subject: [PATCH 048/160] ADD file via upload --- .../graph_ir/npu_loss_scale_ops_declare.h | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.h new file mode 100644 index 00000000000..fdadb5c496b --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/npu_loss_scale_ops_declare.h @@ -0,0 +1,37 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_ +//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/npu_loss_scale_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现NPUGetFloatStatus的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(NPUGetFloatStatus) +DECLARE_OP_USE_OUTPUT(NPUGetFloatStatus) +//对实现NPUAllocFloatStatus的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(NPUAllocFloatStatus) +DECLARE_OP_USE_OUTPUT(NPUAllocFloatStatus) +//对实现NPUClearFloatStatus的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(NPUClearFloatStatus) +DECLARE_OP_USE_OUTPUT(NPUClearFloatStatus) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_NPU_LOSS_SCALE_OPS_DECLARE_H_ -- 2.34.1 From 0abfcedb6357ea43c26cc61256458b8de3448218 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:50:58 +0800 Subject: [PATCH 049/160] ADD file via upload --- op_declare_macro.h | 260 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 op_declare_macro.h diff --git a/op_declare_macro.h b/op_declare_macro.h new file mode 100644 index 00000000000..5e395abeff7 --- /dev/null +++ b/op_declare_macro.h @@ -0,0 +1,260 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_//定义预处理宏CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_ + +//导入系统文件或自定义文件 +#include //导入标准库中的字符串类及相关操作 +#include //提供make_shared指针构建函数模版等 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件，以下同理 +#include "transform/graph_ir/op_adapter.h" +#include "transform/graph_ir/op_adapter_desc.h" +#include "include/transform/graph_ir/op_adapter_map.h" +#include "mindspore/core/base/core_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +/* + 定义宏变量DECLARE_OP_ADAPTER + 函数功能为将收录内容的类型与标准进行对比，后进行空间调整，并用指针input_map_为key存储相应内容 + 具体实现过程： + 判断int与类InputDesc的容量是否小于标准+判断int与类InputDesc是否为可移动构造类型+判断int与类InputDesc是否拥有移动赋值运算符，获得IsFlat + 判断T类型是否为void，若是则返回类型Key，若不是则根据IsFlat的真假返回Key或Key const，并与T类型交换最后返回其值。 + 判断完成后，若结果为真，将范围在4~ 16384的内容从空间分配器中删去 + 若结果为假，建立新的分配器并删除原分配的空间（链表），建立新的内存块并加入到新的分配器中。 + 统计可用元素数量，通过已分配空间计算需要分配的内存大小 + 为新数据创建链表并加入至分配的空间中，元素字节强制对齐。 + 并在建成的堆中分配具体空间 + 使用op_adapter_base.h下ge空间中op类下变量T，利用op_adapter.h中OpAdapter中分配的初始指针作为key储存元素 + string与AttrDesc进行相同操作，用指针attr_map_为key存储相应内容 +*/ +#define DECLARE_OP_ADAPTER(T) \ + using T = ge::op::T; \ + template <> \ + const mindspore::HashMap OpAdapter::input_map_; \ + template <> \ + const mindspore::HashMap OpAdapter::attr_map_; +//定义宏变量DECLARE_OP_USE_OUTPUT +//操作同上，对象变为int+OutputDesc，用指针output_map_为key存储相应内容 +#define DECLARE_OP_USE_OUTPUT(T) \ + template <> \ + const mindspore::HashMap OpAdapter::output_map_; +//定义宏变量DECLARE_OP_USE_ENUM +//操作同上，对象变为int+string，用指针enum_map_为key存储相应内容 +#define DECLARE_OP_USE_ENUM(T) \ + template <> \ + const mindspore::HashMap OpAdapter::enum_map_{}; +//定义宏变量DECLARE_OP_USE_INPUT_ATTR +//操作同上，对象变为unsigned int+AttrDesc，用指针input_attr_map_为key存储相应内容 +#define DECLARE_OP_USE_INPUT_ATTR(T) \ + template <> \ + const mindspore::HashMap OpAdapter::input_attr_map_; +//定义宏变量DECLARE_OP_USE_DYN_INPUT +//操作同上，对象变为int+DynInputDesc，用指针dyn_input_map_为key存储相应内容 +#define DECLARE_OP_USE_DYN_INPUT(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_input_map_; +//定义宏变量DECLARE_OP_USE_DYN_SUBGRAPH +//操作同上，对象变为int+DynSubGraphDesc，用指针dyn_subgraph_map_为key存储相应内容 +#define DECLARE_OP_USE_DYN_SUBGRAPH(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_subgraph_map_; +//定义宏变量DECLARE_OP_USE_DYN_OUTPUT +//操作同上，对象变为int+DynOutputDesc，用指针dyn_output_map_为key存储相应内容 +#define DECLARE_OP_USE_DYN_OUTPUT(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_output_map_; +//定义宏变量INPUT_MAP +//操作同上，对象变为int+InputDesc，用指针input_map_为key存储相应内容 +#define INPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::input_map_ +//定义宏变量EMPTY_INPUT_MAP +//操作同上，对象变为int+InputDesc，不使用指针进行储存 +#define EMPTY_INPUT_MAP mindspore::HashMap() +/* + 定义宏变量INPUT_DESC + 将name变量处理并存入对应InputDesc结构体的相应变量中 + 将name变量内容转为字符串变量并存储至结构体的name变量中 + 引用Operator空间并将指针所指的类转为InputDesc结构体 + 利用指针分别将原指向空间名称、handle变量指向空间+输出空间名称、引用的TensorDesc空间依次存入set_op、set_handle、update_input_desc变量中 +*/ +#define INPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, const OperatorPtr input) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_input_##name(*input); \ + }, \ + [](const OperatorPtr op, const OutHandler& handle) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_input_##name(*(handle.op), handle.out); \ + }, \ + [](const OperatorPtr op, const GeTensorDesc desc) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->update_input_desc_##name(desc); \ + } \ + } +//定义宏变量DYN_INPUT_MAP +//操作同第一个函数，对象变为int+DynInputDesc，用指针dyn_input_map_为key存储相应内容 +#define DYN_INPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_input_map_ +/* + 定义宏变量DYN_INPUT_DESC + 将name变量处理并存入对应DynInputDesc结构体的相应变量中 + 将name变量内容转为字符串变量并存储至结构体的name变量中 + 引用Operator空间并将指针所指的类转为DynInputDesc结构体 + 利用指针分别将携带数字、携带数字+引用Operator空间的内容、数字+handle变量指向空间+输出空间名称依次存入create_dyn_input、set_op、set_handle变量中 +*/ +#define DYN_INPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_input_##name(num); \ + }, \ + [](const OperatorPtr op, unsigned int index, const OperatorPtr input) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_input_##name(index, *input); \ + }, \ + [](const OperatorPtr op, unsigned int index, const OutHandler& handle) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_input_##name(index, *(handle.op), handle.out); \ + } \ + } +//定义宏变量DYN_SUBGRAPH_MAP +//操作同第一个函数，对象变为int+DynInputDesc，用指针dyn_subgraph_map_为key存储相应内容 +#define DYN_SUBGRAPH_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_subgraph_map_ +/* + 定义宏变量DYN_SUBGRAPH_DESC + 将name变量处理并存入对应DynSubgraphDesc结构体的相应变量中 + 将name变量内容转为字符串变量并存储至结构体的name变量中 + 引用Operator空间并将指针所指的类转为DynSubgraphDesc结构体 + 利用指针分别将携带数字、携带数字+Graph空间内容依次存入create_dyn_subgraph、set_subgraph变量中 +*/ +#define DYN_SUBGRAPH_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_subgraph_##name(num); \ + }, \ + [](const OperatorPtr op, unsigned int index, const DfGraphPtr graph) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_subgraph_builder_##name(index, [graph](){return *graph;}); \ + } \ + } +//定义宏变量ATTR_MAP +//操作同第一个函数，对象变为string+AttrDesc，用指针attr_map_为key存储相应内容 +#define ATTR_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::attr_map_ +//定义宏变量EMPTY_ATTR_MAP +//操作同第一个函数，对象变为string+AttrDesc，不使用指针进行储存 +#define EMPTY_ATTR_MAP mindspore::HashMap() +/* + 定义宏变量ATTR_DESC + 将name变量处理并存入对应AttrDesc结构体的相应变量中 + 将name变量内容转为字符串变量并存储至结构体的name变量中 + 引用Operator空间并将指针所指的类转为AttrDesc结构体 + 利用指针将空间存入set_attr变量中 +*/ +#define ATTR_DESC(name, ...) \ + { \ +#name, \ + [](const OperatorPtr op, const ValuePtr& value) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->set_attr_##name(ConvertAny(value, __VA_ARGS__)); \ + } \ + } +//定义宏变量INPUT_ATTR_MAP +//操作同第一个函数，对象变为unsigned int+AttrDesc，用指针input_attr_map_为key存储相应内容 +#define INPUT_ATTR_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::input_attr_map_ +//定义宏变量OUTPUT_MAP +//操作同第一个函数，对象变为int+OutputDesc，用指针output_map_为key存储相应内容 +#define OUTPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::output_map_ +/* + 定义宏变量OUTPUT_DESC + 将name变量处理并存入对应OutputDesc结构体的相应变量中 + 将name变量内容转为字符串变量并存储至结构体的name变量中 + 引用Operator空间并将指针所指的类转为OutputDesc结构体 + 利用指针将引用的TensorDesc空间存入update_out_desc变量中 +*/ +#define OUTPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, const GeTensorDesc desc) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->update_output_desc_##name(desc); \ + } \ + } +//定义宏变量DYN_OUTPUT_MAP +//操作同第一个函数，对象变为int+DynOutputDesc，用指针dyn_output_map_为key存储相应内容 +#define DYN_OUTPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_output_map_ +/* + 定义宏变量DYN_OUTPUT_DESC + 将name变量处理并存入对应DynOutputDesc结构体的相应变量中 + 将name变量内容转为字符串变量并存储至结构体的name变量中 + 引用Operator空间并将指针所指的类转为DynOutputDesc结构体 + 利用指针将携带数字存入create_dyn_output变量中 +*/ +#define DYN_OUTPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \ + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_output_##name(num); \ + } \ + } +//定义宏变量ADPT_DESC_ONE +//分配并构造类型为OpAdapterDesc的对象，将类型为T、指向OpAdapter空间的指针传递给其构造函数，然后返回类型为shared_ptr的对象，该对象拥有并存储指向它的指针 +#define ADPT_DESC_ONE(T) std::make_shared(std::make_shared>()) +//定义宏变量ADPT_DESC_TWO +//分配并构造类型为OpAdapterDesc的对象，将类型为T、指向OpAdapter空间的指针+类型为I、指向OpAdapter空间的指针传递给其构造函数，然后返回类型为shared_ptr的对象，该对象拥有并存储指向它的指针 +#define ADPT_DESC_TWO(T, I) \ + std::make_shared(std::make_shared>(), std::make_shared>()) +//定义宏变量GET_DESC +//承担各形参变量的支架函数 +#define GET_MACRO(_1, _2, DESC, ...) DESC +//定义宏变量ADPT_DESC +//将GET_MACRO宏变量中嵌套ADPT_DESC_ONE、ADPT_DESC_TWO等宏变量，构造对应空间指针 +#define ADPT_DESC(...) GET_MACRO(__VA_ARGS__, ADPT_DESC_TWO, ADPT_DESC_ONE, ...)(__VA_ARGS__) +/* + 定义宏变量REG_ADPT_DESC + 创建结构体RegAdptDesc+name + 创建公共型函数RegAdptDesc+name：将访问OpAdapterDesc空间的内容与string进行对比，后进行空间调整，并用指针adpt_desc指针存储相应内容 + 创建私密性整形变量：ph_{0} + 将其结构体自定义命名为g_reg_adpt_desc_+name +*/ +#define REG_ADPT_DESC(name, name_str, adpt_desc) \ + static struct RegAdptDesc##name { \ + public: \ + RegAdptDesc##name() { OpAdapterMap::get()[name_str] = adpt_desc; } \ + \ + private: \ + int ph_{0}; \ + } g_reg_adpt_desc_##name; +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_ -- 2.34.1 From 0dd4e350beeb41a764cd30a0c8e5c2204ec8d161 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:51:16 +0800 Subject: [PATCH 050/160] ADD file via upload --- .../transform/graph_ir/pad_ops_declare.cc | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.cc new file mode 100644 index 00000000000..2e599d79196 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.cc @@ -0,0 +1,86 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/pad_ops_declare.h"//按照路径寻找以下文件，导入到本文件 +#include //提供vector数组构建函数模版等 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// PadD +INPUT_MAP(PadD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入NPUGetFloatStatus对应空间内并用input_map指针保存 +ATTR_MAP(PadD) = {{"paddings", ATTR_DESC(paddings, AnyTraits>>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入PadD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(PadD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PadD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(PadD, kNamePadD, ADPT_DESC(PadD))//构造指向PadD的指针并储存，创建结构体RegAdptDescPadD + +// Pad +INPUT_MAP(Pad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +//将变量x、paddings处理并存入对应InputDesc结构体的相应变量中，存入Pad对应空间内并用input_map指针保存 +ATTR_MAP(Pad) = EMPTY_ATTR_MAP;//将空变量存入Pad对应空间并用attr_map_指针保存 +OUTPUT_MAP(Pad) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Pad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Pad, kNamePadV1, ADPT_DESC(Pad))//构造指向Pad的指针并储存，创建结构体RegAdptDescPad + +// BroadcastToD +INPUT_MAP(BroadcastToD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入BroadcastToD对应空间内并用input_map指针保存 +ATTR_MAP(BroadcastToD) = {{"shape", ATTR_DESC(shape, AnyTraits(), AnyTraits>())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入BroadcastToD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(BroadcastToD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入BroadcastToD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BroadcastToD, kNameBroadcastTo, ADPT_DESC(BroadcastToD))//构造指向BroadcastToD的指针并储存，创建结构体RegAdptDescBroadcastToD + +// Diag +INPUT_MAP(Diag) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Diag对应空间内并用input_map指针保存 +ATTR_MAP(Diag) = EMPTY_ATTR_MAP;//将空变量存入Diag对应空间并用attr_map_指针保存 +OUTPUT_MAP(Diag) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Diag对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Diag, kNameDiag, ADPT_DESC(Diag))//构造指向Diag的指针并储存，创建结构体RegAdptDescDiag + +// FillD +INPUT_MAP(FillD) = {{1, INPUT_DESC(value)}};//将变量value处理并存入对应InputDesc结构体的相应变量中，存入FillD对应空间内并用input_map指针保存 +ATTR_MAP(FillD) = {{"dims", ATTR_DESC(dims, AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入FillD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(FillD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入FillD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(FillD, kNameFillD, ADPT_DESC(FillD))//构造指向FillD的指针并储存，创建结构体RegAdptDescFillD + +// Fill +INPUT_MAP(Fill) = {{1, INPUT_DESC(dims)}, {2, INPUT_DESC(value)}}; +//将变量dims、value处理并存入对应InputDesc结构体的相应变量中，存入Fill对应空间内并用input_map指针保存 +ATTR_MAP(Fill) = EMPTY_ATTR_MAP;//将空变量存入Fill对应空间并用attr_map_指针保存 +OUTPUT_MAP(Fill) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Fill对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Fill, kNameFillV1, ADPT_DESC(Fill))//构造指向Fill的指针并储存，创建结构体RegAdptDescFill + +// PadV3 +INPUT_MAP(PadV3) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}, {3, INPUT_DESC(constant_values)}}; +//将变量x、paddings、constant_values处理并存入对应InputDesc结构体的相应变量中，存入PadV3对应空间内并用input_map指针保存 +ATTR_MAP(PadV3) = {{"mode", ATTR_DESC(mode, AnyTraits())}, + {"pad_contiguous", ATTR_DESC(paddings_contiguous, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入PadV3对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(PadV3) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PadV3对应空间内并用output_map_指针保存 +REG_ADPT_DESC(PadV3, kNamePadV3, ADPT_DESC(PadV3))//构造指向PadV3的指针并储存，创建结构体RegAdptDescPadV3 + +// PadV2 +INPUT_MAP(PadV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}, {3, INPUT_DESC(constant_values)}}; +//将变量x、paddings、constant_values处理并存入对应InputDesc结构体的相应变量中，存入PadV2对应空间内并用input_map指针保存 +ATTR_MAP(PadV2) = EMPTY_ATTR_MAP;//将空变量存入PadV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(PadV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入PadV2对应空间内并用output_map_指针保存 +REG_ADPT_DESC(PadV2, kNamePadV2, ADPT_DESC(PadV2))//构造指向PadV2的指针并储存，创建结构体RegAdptDescPadV2 +} // namespace mindspore::transform -- 2.34.1 From d20f70d889b91b92067b772fd63663766076717b Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:51:40 +0800 Subject: [PATCH 051/160] ADD file via upload --- .../transform/graph_ir/pad_ops_declare.h | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.h new file mode 100644 index 00000000000..4df4bfab819 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/pad_ops_declare.h @@ -0,0 +1,51 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/pad_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现PadD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(PadD) +DECLARE_OP_USE_OUTPUT(PadD) +//对实现Pad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Pad) +DECLARE_OP_USE_OUTPUT(Pad) +//对实现BroadcastToD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BroadcastToD) +DECLARE_OP_USE_OUTPUT(BroadcastToD) +//对实现Diag的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Diag) +DECLARE_OP_USE_OUTPUT(Diag) +//对实现FillD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(FillD) +DECLARE_OP_USE_OUTPUT(FillD) +//对实现Fill的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Fill) +DECLARE_OP_USE_OUTPUT(Fill) +//对实现PadV3的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(PadV3) +DECLARE_OP_USE_OUTPUT(PadV3) +//对实现PadV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(PadV2) +DECLARE_OP_USE_OUTPUT(PadV2) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_PAD_OPS_DECLARE_H_ -- 2.34.1 From e271f89b8c30cea89276019be6651aecf5b423fb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:51:57 +0800 Subject: [PATCH 052/160] ADD file via upload --- .../graph_ir/quantize_ops_declare.cc | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.cc new file mode 100644 index 00000000000..32e176950d1 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/quantize_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// AscendQuant +INPUT_MAP(AscendQuant) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入AscendQuant对应空间内并用input_map指针保存 +ATTR_MAP(AscendQuant) = {{"scale", ATTR_DESC(scale, AnyTraits())}, + {"offset", ATTR_DESC(offset, AnyTraits())}, + {"sqrt_mode", ATTR_DESC(sqrt_mode, AnyTraits())}, + {"round_mode", ATTR_DESC(round_mode, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入AscendQuant对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(AscendQuant) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AscendQuant对应空间内并用output_map_指针保存 +REG_ADPT_DESC(AscendQuant, kNameAscendQuant, ADPT_DESC(AscendQuant))//构造指向AscendQuant的指针并储存，创建结构体RegAdptDescAscendQuant + +// AscendDequant +INPUT_MAP(AscendDequant) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(deq_scale)}}; +//将变量x、deq_scale处理并存入对应InputDesc结构体的相应变量中，存入AscendDequant对应空间内并用input_map指针保存 +ATTR_MAP(AscendDequant) = {{"sqrt_mode", ATTR_DESC(sqrt_mode, AnyTraits())}, + {"relu_flag", ATTR_DESC(relu_flag, AnyTraits())}, + {"dtype", ATTR_DESC(dtype, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入AscendDequant对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(AscendDequant) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入AscendDequant对应空间内并用output_map_指针保存 +REG_ADPT_DESC(AscendDequant, kNameAscendDequant, ADPT_DESC(AscendDequant))//构造指向AscendDequant的指针并储存，创建结构体RegAdptDescAscendDequant +} // namespace mindspore::transform -- 2.34.1 From 0f1be59be5e8282e5f0a80b0d2cbed9d796ec4d2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:52:13 +0800 Subject: [PATCH 053/160] ADD file via upload --- .../transform/graph_ir/quantize_ops_declare.h | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.h new file mode 100644 index 00000000000..63d00305ba0 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/quantize_ops_declare.h @@ -0,0 +1,33 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/quantize_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现AscendQuant的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(AscendQuant) +DECLARE_OP_USE_OUTPUT(AscendQuant) +//对实现AscendDequant的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(AscendDequant) +DECLARE_OP_USE_OUTPUT(AscendDequant) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_ -- 2.34.1 From 6b3130ad0bfa1c67bbb78376833b3d056f17bda9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:52:29 +0800 Subject: [PATCH 054/160] ADD file via upload --- .../transform/graph_ir/random_ops_declare.cc | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.cc new file mode 100644 index 00000000000..e5e76bb90af --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.cc @@ -0,0 +1,59 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/random_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// DropOutGenMask +INPUT_MAP(DropOutGenMask) = {{1, INPUT_DESC(shape)}, {2, INPUT_DESC(prob)}}; +//将变量shape、prob处理并存入对应InputDesc结构体的相应变量中，存入DropOutGenMask对应空间内并用input_map指针保存 +ATTR_MAP(DropOutGenMask) = {{"Seed0", ATTR_DESC(seed, AnyTraits())}, + {"Seed1", ATTR_DESC(seed2, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DropOutGenMask对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DropOutGenMask) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入DropOutGenMask对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DropOutGenMask, prim::kPrimDropoutGenMask->name(), ADPT_DESC(DropOutGenMask)) +//构造指向DropOutGenMask的指针并储存，创建结构体RegAdptDescDropOutGenMask + +// LinSpace +INPUT_MAP(LinSpace) = {{1, INPUT_DESC(start)}, {2, INPUT_DESC(stop)}, {3, INPUT_DESC(num)}}; +//将变量start、stop、num处理并存入对应InputDesc结构体的相应变量中，存入LinSpace对应空间内并用input_map指针保存 +ATTR_MAP(LinSpace) = EMPTY_ATTR_MAP;//将空变量存入LinSpace对应空间并用attr_map_指针保存 +OUTPUT_MAP(LinSpace) = {{0, OUTPUT_DESC(output)}};//将变量output处理并存入对应OutputDesc结构体的相应变量中，存入LinSpace对应空间内并用output_map_指针保存 +REG_ADPT_DESC(LinSpace, kNameLinSpace, ADPT_DESC(LinSpace))//构造指向LinSpace的指针并储存，创建结构体RegAdptDescLinSpace + +// RandomChoiceWithMask +INPUT_MAP(RandomChoiceWithMask) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入RandomChoiceWithMask对应空间内并用input_map指针保存 +ATTR_MAP(RandomChoiceWithMask) = {{"count", ATTR_DESC(count, AnyTraits())}, + {"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入RandomChoiceWithMask对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(RandomChoiceWithMask) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}}; +//将变量y、mask处理并存入对应OutputDesc结构体的相应变量中，存入RandomChoiceWithMask对应空间内并用output_map_指针保存 +REG_ADPT_DESC(RandomChoiceWithMask, kNameRandomChoiceWithMask, ADPT_DESC(RandomChoiceWithMask)) +//构造指向RandomChoiceWithMask的指针并储存，创建结构体RegAdptDescRandomChoiceWithMask + +// TruncatedNormal +INPUT_MAP(TruncatedNormal) = {{1, INPUT_DESC(shape)}};//将变量shape处理并存入对应InputDesc结构体的相应变量中，存入TruncatedNormal对应空间内并用input_map指针保存 +ATTR_MAP(TruncatedNormal) = {{"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入TruncatedNormal对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(TruncatedNormal) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入TruncatedNormal对应空间内并用output_map_指针保存 +REG_ADPT_DESC(TruncatedNormal, kNameTruncatedNormal, ADPT_DESC(TruncatedNormal)) +//构造指向TruncatedNormal的指针并储存，创建结构体RegAdptDescTruncatedNormal +} // namespace mindspore::transform -- 2.34.1 From b7c6f3b41258d5533833595fb97543889d5600ef Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:52:57 +0800 Subject: [PATCH 055/160] ADD file via upload --- .../transform/graph_ir/random_ops_declare.h | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.h new file mode 100644 index 00000000000..80d26353ea2 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/random_ops_declare.h @@ -0,0 +1,39 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RANDOM_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RANDOM_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_QUANTIZE_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/random_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现DropOutGenMask的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DropOutGenMask) +DECLARE_OP_USE_OUTPUT(DropOutGenMask) +//对实现LinSpace的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(LinSpace) +DECLARE_OP_USE_OUTPUT(LinSpace) +//对实现RandomChoiceWithMask的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(RandomChoiceWithMask) +DECLARE_OP_USE_OUTPUT(RandomChoiceWithMask) +//对实现TruncatedNormal的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(TruncatedNormal) +DECLARE_OP_USE_OUTPUT(TruncatedNormal) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RANDOM_OPS_DECLARE_H_ -- 2.34.1 From e45fb7005a404c43c0c57796b2e26c0e617e3bc0 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:53:31 +0800 Subject: [PATCH 056/160] ADD file via upload --- .../transform/graph_ir/reduce_ops_declare.cc | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.cc new file mode 100644 index 00000000000..384fb001c0c --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.cc @@ -0,0 +1,144 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/reduce_ops_declare.h"//按照路径寻找以下文件，导入到本文件 +#include //提供vector数组构建函数模版等 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// BNTrainingReduce +INPUT_MAP(BNTrainingReduce) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入BNTrainingReduce对应空间内并用input_map指针保存 +ATTR_MAP(BNTrainingReduce) = EMPTY_ATTR_MAP;//将空变量存入BNTrainingReduce对应空间并用attr_map_指针保存 +OUTPUT_MAP(BNTrainingReduce) = {{0, OUTPUT_DESC(sum)}, {1, OUTPUT_DESC(square_sum)}}; +//将变量sum、square_sum处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingReduce对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BNTrainingReduce, kNameBNTrainingReduce, ADPT_DESC(BNTrainingReduce)) +//构造指向BNTrainingReduce的指针并储存，创建结构体RegAdptDescBNTrainingReduce + +// BNTrainingReduceGrad +INPUT_MAP(BNTrainingReduceGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(diff_scale)}, + {4, INPUT_DESC(diff_offset)}, {5, INPUT_DESC(scale)}, {6, INPUT_DESC(batch_mean)}, + {7, INPUT_DESC(batch_variance)}}; +//将变量grads、x、diff_scale、diff_offset、scale、batch_mean、batch_variance处理并存入对应InputDesc结构体的相应变量中 +//存入BNTrainingReduceGrad对应空间内并用input_map指针保存 +ATTR_MAP(BNTrainingReduceGrad) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入BNTrainingReduceGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(BNTrainingReduceGrad) = {{0, OUTPUT_DESC(y)}}; +//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingReduceGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BNTrainingReduceGrad, kNameBNTrainingReduceGrad, ADPT_DESC(BNTrainingReduceGrad)) +//构造指向BNTrainingReduceGrad的指针并储存，创建结构体RegAdptDescBNTrainingReduceGrad + +// BNTrainingUpdate +INPUT_MAP(BNTrainingUpdate) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(sum)}, {3, INPUT_DESC(square_sum)}, + {4, INPUT_DESC(scale)}, {5, INPUT_DESC(offset)}, {6, INPUT_DESC(mean)}, + {7, INPUT_DESC(variance)}}; +//将变量x、sum、square_sum、scale、offset、mean、variance处理并存入对应InputDesc结构体的相应变量中 +//存入BNTrainingUpdate对应空间内并用input_map指针保存 +ATTR_MAP(BNTrainingUpdate) = {{"factor", ATTR_DESC(factor, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入BNTrainingUpdate对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(BNTrainingUpdate) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(mean)}, + {2, OUTPUT_DESC(variance)}, + {3, OUTPUT_DESC(batch_mean)}, + {4, OUTPUT_DESC(batch_variance)}}; +//将变量y、mean、variance、batch_mean、batch_variance处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingUpdate对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BNTrainingUpdate, kNameBNTrainingUpdate, ADPT_DESC(BNTrainingUpdate)) +//构造指向BNTrainingUpdate的指针并储存，创建结构体RegAdptDescBNTrainingUpdate + +// BNTrainingUpdateGrad +INPUT_MAP(BNTrainingUpdateGrad) = { + {1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(batch_mean)}, {4, INPUT_DESC(batch_variance)}}; +//将变量grads、x、batch_mean、batch_variance处理并存入对应InputDesc结构体的相应变量中 +//存入BNTrainingUpdateGrad对应空间内并用input_map指针保存 +ATTR_MAP(BNTrainingUpdateGrad) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入BNTrainingUpdateGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(BNTrainingUpdateGrad) = {{0, OUTPUT_DESC(diff_scale)}, {1, OUTPUT_DESC(diff_offset)}}; +//将变量diff_scale、diff_offset处理并存入对应OutputDesc结构体的相应变量中，存入BNTrainingUpdateGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BNTrainingUpdateGrad, kNameBNTrainingUpdateGrad, ADPT_DESC(BNTrainingUpdateGrad)) +//构造指向BNTrainingUpdateGrad的指针并储存，创建结构体RegAdptDescBNTrainingUpdateGrad + +// ReduceAnyD +INPUT_MAP(ReduceAnyD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceAnyD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceAnyD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceAnyD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceAnyD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceAnyD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceAnyD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceAnyD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceAnyD, kNameReduceAnyD, ADPT_DESC(ReduceAnyD))//构造指向ReduceAnyD的指针并储存，创建结构体RegAdptDescReduceAnyD + +// ReduceSumD +INPUT_MAP(ReduceSumD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceSumD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceSumD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceSumD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceSumD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceSumD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceSumD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceSumD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceSumD, prim::kPrimReduceSum->name(), ADPT_DESC(ReduceSumD))//构造指向ReduceSumD的指针并储存，创建结构体RegAdptDescReduceSumD + +// ReduceProdD +INPUT_MAP(ReduceProdD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceProdD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceProdD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceProdD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceProdD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceProdD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceProdD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceProdD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceProdD, kNameReduceProd, ADPT_DESC(ReduceProdD))//构造指向ReduceProdD的指针并储存，创建结构体RegAdptDescReduceProdD + +// ReduceAllD +INPUT_MAP(ReduceAllD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceAllD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceAllD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceAllD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceAllD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceAllD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceAllD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceAllD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceAllD, prim::kPrimReduceAll->name(), ADPT_DESC(ReduceAllD))//构造指向ReduceAllD的指针并储存，创建结构体RegAdptDescReduceAllD + +// ReduceMeanD +INPUT_MAP(ReduceMeanD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceMeanD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceMeanD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceMeanD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceMeanD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceMeanD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceMeanD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceMeanD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceMeanD, prim::kPrimReduceMean->name(), ADPT_DESC(ReduceMeanD))//构造指向ReduceMeanD的指针并储存，创建结构体RegAdptDescReduceMeanD + +// ReduceMinD +INPUT_MAP(ReduceMinD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceMinD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceMinD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceMinD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceMinD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceMinD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceMinD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceMinD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceMinD, prim::kPrimReduceMin->name(), ADPT_DESC(ReduceMinD))//构造指向ReduceMinD的指针并储存，创建结构体RegAdptDescReduceMinD + +// ReduceMaxD +INPUT_MAP(ReduceMaxD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReduceMaxD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ReduceMaxD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}};//对实现ReduceMaxD的对象转换后转移出相应存储空间 +ATTR_MAP(ReduceMaxD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReduceMaxD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ReduceMaxD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReduceMaxD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReduceMaxD, prim::kPrimReduceMax->name(), ADPT_DESC(ReduceMaxD))//构造指向ReduceMaxD的指针并储存，创建结构体RegAdptDescReduceMaxD +} // namespace mindspore::transform -- 2.34.1 From 1bc31be401e5176f2a4d4a93f3b6197efb84b3c4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:53:49 +0800 Subject: [PATCH 057/160] ADD file via upload --- .../transform/graph_ir/reduce_ops_declare.h | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.h new file mode 100644 index 00000000000..f68c87c50fc --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/reduce_ops_declare.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/reduce_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现ReduceMean的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceMean) +//对实现ReduceMinD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceMinD) +DECLARE_OP_USE_INPUT_ATTR(ReduceMinD) +DECLARE_OP_USE_OUTPUT(ReduceMinD) +//对实现ReduceMaxD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceMaxD) +DECLARE_OP_USE_INPUT_ATTR(ReduceMaxD) +DECLARE_OP_USE_OUTPUT(ReduceMaxD) +//对实现ReduceAllD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceAllD) +DECLARE_OP_USE_INPUT_ATTR(ReduceAllD) +DECLARE_OP_USE_OUTPUT(ReduceAllD) +//对实现BNTrainingReduce的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BNTrainingReduce) +DECLARE_OP_USE_OUTPUT(BNTrainingReduce) +//对实现BNTrainingReduceGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BNTrainingReduceGrad) +DECLARE_OP_USE_OUTPUT(BNTrainingReduceGrad) +//对实现BNTrainingUpdate的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BNTrainingUpdate) +DECLARE_OP_USE_OUTPUT(BNTrainingUpdate) +//对实现BNTrainingUpdateGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BNTrainingUpdateGrad) +DECLARE_OP_USE_OUTPUT(BNTrainingUpdateGrad) +//对实现ReduceSumD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceSumD) +DECLARE_OP_USE_INPUT_ATTR(ReduceSumD) +DECLARE_OP_USE_OUTPUT(ReduceSumD) +//对实现ReduceAnyD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceAnyD) +DECLARE_OP_USE_INPUT_ATTR(ReduceAnyD) +DECLARE_OP_USE_OUTPUT(ReduceAnyD) +//对实现ReduceMeanD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceMeanD) +DECLARE_OP_USE_INPUT_ATTR(ReduceMeanD) +DECLARE_OP_USE_OUTPUT(ReduceMeanD) +//对实现ReduceProdD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReduceProdD) +DECLARE_OP_USE_INPUT_ATTR(ReduceProdD) +DECLARE_OP_USE_OUTPUT(ReduceProdD) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_REDUCE_OPS_DECLARE_H_ -- 2.34.1 From 6c3ac2d7dfe19024f1395d7a3f3e378a7cb36c0a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:53:58 +0800 Subject: [PATCH 058/160] ADD file via upload --- rnn_declare.cc | 183 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 rnn_declare.cc diff --git a/rnn_declare.cc b/rnn_declare.cc new file mode 100644 index 00000000000..54a9f729d89 --- /dev/null +++ b/rnn_declare.cc @@ -0,0 +1,183 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/rnn_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// BasicLSTMCell +INPUT_MAP(BasicLSTMCell) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(h)}, {3, INPUT_DESC(c)}, {4, INPUT_DESC(w)}, {5, INPUT_DESC(b)}}; +//将变量x、h、c、w、b处理并存入对应InputDesc结构体的相应变量中，存入BNTrainingReduce对应空间内并用input_map指针保存 +ATTR_MAP(BasicLSTMCell) = {{"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}, + {"state_is_tuple", ATTR_DESC(state_is_tuple, AnyTraits())}, + {"activation", ATTR_DESC(activation, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入BasicLSTMCell对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(BasicLSTMCell) = {{0, OUTPUT_DESC(ct)}, {1, OUTPUT_DESC(ht)}, {2, OUTPUT_DESC(it)}, {3, OUTPUT_DESC(jt)}, + {4, OUTPUT_DESC(ft)}, {5, OUTPUT_DESC(ot)}, {6, OUTPUT_DESC(tanhct)}}; +//将变量ct、ht、it、jt、ft、ot、tanhct处理并存入对应OutputDesc结构体的相应变量中，存入BasicLSTMCell对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BasicLSTMCell, kNameBasicLSTMCell, ADPT_DESC(BasicLSTMCell)) +//构造指向BasicLSTMCell的指针并储存，创建结构体RegAdptDescBasicLSTMCell + +// BasicLSTMCellInputGrad +INPUT_MAP(BasicLSTMCellInputGrad) = {{1, INPUT_DESC(dgate)}, {2, INPUT_DESC(w)}}; +//将变量dgate、w处理并存入对应InputDesc结构体的相应变量中，存入BasicLSTMCellInputGrad对应空间内并用input_map指针保存 +ATTR_MAP(BasicLSTMCellInputGrad) = {{"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入BasicLSTMCellInputGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(BasicLSTMCellInputGrad) = {{0, OUTPUT_DESC(dxt)}, {1, OUTPUT_DESC(dht)}}; +//将变量dxt、dht处理并存入对应OutputDesc结构体的相应变量中，存入BasicLSTMCellInputGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BasicLSTMCellInputGrad, kNameBasicLSTMCellInputGrad, ADPT_DESC(BasicLSTMCellInputGrad)) +//构造指向BasicLSTMCellInputGrad的指针并储存，创建结构体RegAdptDescBasicLSTMCellInputGrad + +// BasicLSTMCellWeightGrad +INPUT_MAP(BasicLSTMCellWeightGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(h)}, {3, INPUT_DESC(dgate)}}; +//将变量x、h、dgate处理并存入对应InputDesc结构体的相应变量中，存入BasicLSTMCellWeightGrad对应空间内并用input_map指针保存 +ATTR_MAP(BasicLSTMCellWeightGrad) = EMPTY_ATTR_MAP;//将空变量存入BasicLSTMCellWeightGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(BasicLSTMCellWeightGrad) = {{0, OUTPUT_DESC(dw)}, {1, OUTPUT_DESC(db)}}; +//将变量dw、db处理并存入对应OutputDesc结构体的相应变量中，存入BasicLSTMCellWeightGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BasicLSTMCellWeightGrad, kNameBasicLSTMCellWeightGrad, ADPT_DESC(BasicLSTMCellWeightGrad)) +//构造指向BasicLSTMCellWeightGrad的指针并储存，创建结构体RegAdptDescBasicLSTMCellWeightGrad + +// BasicLSTMCellCStateGrad +INPUT_MAP(BasicLSTMCellCStateGrad) = {{1, INPUT_DESC(c)}, {2, INPUT_DESC(dht)}, {3, INPUT_DESC(dct)}, + {4, INPUT_DESC(it)}, {5, INPUT_DESC(jt)}, {6, INPUT_DESC(ft)}, + {7, INPUT_DESC(ot)}, {8, INPUT_DESC(tanhct)}}; +//将变量c、dht、dct、it、jt、ft、ot、tanhct处理并存入对应InputDesc结构体的相应变量中，存入BasicLSTMCellCStateGrad对应空间内并用input_map指针保存 +ATTR_MAP(BasicLSTMCellCStateGrad) = {{"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}, + {"activation", ATTR_DESC(activation, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入BasicLSTMCellCStateGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(BasicLSTMCellCStateGrad) = {{0, OUTPUT_DESC(dgate)}, {1, OUTPUT_DESC(dct_1)}}; +//将变量dgate、dct_1处理并存入对应OutputDesc结构体的相应变量中，存入BasicLSTMCellCStateGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BasicLSTMCellCStateGrad, kNameBasicLSTMCellCStateGrad, ADPT_DESC(BasicLSTMCellCStateGrad)) +//构造指向BasicLSTMCellCStateGrad的指针并储存，创建结构体RegAdptDescBasicLSTMCellCStateGrad + +// LSTMInputGrad +INPUT_MAP(LSTMInputGrad) = {{1, INPUT_DESC(w)}, {2, INPUT_DESC(init_c)}, {3, INPUT_DESC(c)}, {4, INPUT_DESC(dy)}, + {5, INPUT_DESC(dh)}, {6, INPUT_DESC(dc)}, {7, INPUT_DESC(i)}, {8, INPUT_DESC(j)}, + {9, INPUT_DESC(f)}, {10, INPUT_DESC(o)}, {11, INPUT_DESC(tanhct)}}; +//将变量w、init_c、c、dy、dh、dc、i、j、f、o、tanhct处理并存入对应InputDesc结构体的相应变量中，存入LSTMInputGrad对应空间内并用input_map指针保存 +ATTR_MAP(LSTMInputGrad) = EMPTY_ATTR_MAP;//将空变量存入LSTMInputGrad对应空间并用attr_map_指针保存 +OUTPUT_MAP(LSTMInputGrad) = { + {0, OUTPUT_DESC(dx)}, {1, OUTPUT_DESC(dh_prev)}, {2, OUTPUT_DESC(dc_prev)}, {4, OUTPUT_DESC(dgate)}}; +//将变量dx、dh_prev、dc_prev、dgate处理并存入对应OutputDesc结构体的相应变量中，存入LSTMInputGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(LSTMInputGrad, kNameLSTMInputGrad, ADPT_DESC(LSTMInputGrad)) +//构造指向LSTMInputGrad的指针并储存，创建结构体RegAdptDescLSTMInputGrad + +// DynamicRNN +INPUT_MAP(DynamicRNN) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, + {4, INPUT_DESC(seq_length)}, {5, INPUT_DESC(init_h)}, {6, INPUT_DESC(init_c)}, + {7, INPUT_DESC(wci)}, {8, INPUT_DESC(wcf)}, {9, INPUT_DESC(wco)}, + {10, INPUT_DESC(mask)}}; +//将变量x、w、c、b、seq_length、init_h、init_c、wci、wcf、wco、mask处理并存入对应InputDesc结构体的相应变量中，存入DynamicRNN对应空间内并用input_map指针保存 +ATTR_MAP(DynamicRNN) = {{"cell_type", ATTR_DESC(cell_type, AnyTraits())}, + {"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"use_peephole", ATTR_DESC(use_peephole, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"ivation", ATTR_DESC(activation, AnyTraits())}, + {"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DynamicRNN对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DynamicRNN) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(output_h)}, {2, OUTPUT_DESC(output_c)}, + {3, OUTPUT_DESC(i)}, {4, OUTPUT_DESC(j)}, {5, OUTPUT_DESC(f)}, + {6, OUTPUT_DESC(o)}, {7, OUTPUT_DESC(tanhc)}}; +//将变量y、output_h、output_c、i、j、f、o、tanhc处理并存入对应OutputDesc结构体的相应变量中，存入DynamicRNN对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DynamicRNN, kNameDynamicRNN, ADPT_DESC(DynamicRNN)) +//构造指向DynamicRNN的指针并储存，创建结构体RegAdptDescDynamicRNN + +// DynamicRNNGrad +INPUT_MAP(DynamicRNNGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, {4, INPUT_DESC(y)}, + {5, INPUT_DESC(init_h)}, {6, INPUT_DESC(init_c)}, {7, INPUT_DESC(h)}, {8, INPUT_DESC(c)}, + {9, INPUT_DESC(dy)}, {10, INPUT_DESC(dh)}, {11, INPUT_DESC(dc)}, {12, INPUT_DESC(i)}, + {13, INPUT_DESC(j)}, {14, INPUT_DESC(f)}, {15, INPUT_DESC(o)}, {16, INPUT_DESC(tanhct)}}; +//将变量x、w、c、b、y、init_h、init_c、h、c、dy、dh、dc、i、j、f、o、tanhct处理并存入对应InputDesc结构体的相应变量中，存入DynamicRNNGrad对应空间内并用input_map指针保存 +ATTR_MAP(DynamicRNNGrad) = {{"cell_type", ATTR_DESC(cell_type, AnyTraits())}, + {"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"use_peephole", ATTR_DESC(use_peephole, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DynamicRNNGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DynamicRNNGrad) = {{0, OUTPUT_DESC(dw)}, + {1, OUTPUT_DESC(db)}, + {2, OUTPUT_DESC(dx)}, + {3, OUTPUT_DESC(dh_prev)}, + {4, OUTPUT_DESC(dc_prev)}}; +//将变量dw、db、dx、dh_prev、dc_prev处理并存入对应OutputDesc结构体的相应变量中，存入DynamicRNNGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DynamicRNNGrad, kNameDynamicRNNGrad, ADPT_DESC(DynamicRNNGrad)) +//构造指向DynamicRNNGrad的指针并储存，创建结构体RegAdptDescDynamicRNNGrad + +// DynamicGRUV2 +INPUT_MAP(DynamicGRUV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(weight_input)}, {3, INPUT_DESC(weight_hidden)}, + {4, INPUT_DESC(bias_input)}, {5, INPUT_DESC(bias_hidden)}, {6, INPUT_DESC(seq_length)}, + {7, INPUT_DESC(init_h)}}; +//将变量x、weight_input、weight_hidden、bias_input、bias_hidden、seq_length、init_h处理并存入对应InputDesc结构体的相应变量中 +//存入DynamicGRUV2对应空间内并用input_map指针保存 +ATTR_MAP(DynamicGRUV2) = {{"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"activation", ATTR_DESC(activation, AnyTraits())}, + {"gate_order", ATTR_DESC(gate_order, AnyTraits())}, + {"reset_after", ATTR_DESC(reset_after, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DynamicGRUV2对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DynamicGRUV2) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(output_h)}, {2, OUTPUT_DESC(update)}, + {3, OUTPUT_DESC(reset)}, {4, OUTPUT_DESC(new)}, {5, OUTPUT_DESC(hidden_new)}}; +//将变量y、output_h、update、reset、new、hidden_new处理并存入对应OutputDesc结构体的相应变量中，存入DynamicGRUV2对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DynamicGRUV2, kNameDynamicGRUV2, ADPT_DESC(DynamicGRUV2)) +//构造指向DynamicGRUV2的指针并储存，创建结构体RegAdptDescDynamicGRUV2 + +// DynamicGRUV2Grad +INPUT_MAP(DynamicGRUV2Grad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(weight_input)}, {3, INPUT_DESC(weight_hidden)}, + {4, INPUT_DESC(y)}, {5, INPUT_DESC(init_h)}, {6, INPUT_DESC(h)}, + {7, INPUT_DESC(dy)}, {8, INPUT_DESC(dh)}, {9, INPUT_DESC(update)}, + {10, INPUT_DESC(reset)}, {11, INPUT_DESC(new)}, {12, INPUT_DESC(hidden_new)}, + {13, INPUT_DESC(seq_length)}, {14, INPUT_DESC(mask)}}; +//将变量x、weight_input、weight_hidden、y、init_h、h、dy、dh、update、reset、new、hidden_new、seq_length、mask处理并存入对应InputDesc结构体的相应变量中 +//存入DynamicGRUV2Grad对应空间内并用input_map指针保存 +ATTR_MAP(DynamicGRUV2Grad) = {{"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"gate_order", ATTR_DESC(gate_order, AnyTraits())}, + {"reset_after", ATTR_DESC(reset_after, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DynamicGRUV2Grad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DynamicGRUV2Grad) = {{0, OUTPUT_DESC(dw_input)}, {1, OUTPUT_DESC(dw_hidden)}, {2, OUTPUT_DESC(db_input)}, + {3, OUTPUT_DESC(db_hidden)}, {4, OUTPUT_DESC(dx)}, {5, OUTPUT_DESC(dh_prev)}}; +//将变量dw_input、dw_hidden、db_input、db_hidden、dx、dh_prev处理并存入对应OutputDesc结构体的相应变量中，存入DynamicGRUV2Grad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DynamicGRUV2Grad, kNameDynamicGRUV2Grad, ADPT_DESC(DynamicGRUV2Grad)) +//构造指向DynamicGRUV2Grad的指针并储存，创建结构体RegAdptDescDynamicGRUV2Grad +} // namespace mindspore::transform -- 2.34.1 From 450b4b9ce823abc813501f5db463dceb3c51983a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:54:21 +0800 Subject: [PATCH 059/160] ADD file via upload --- .../ccsrc/transform/graph_ir/rnn_declare.h | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/rnn_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/rnn_declare.h b/mindspore2022/ccsrc/transform/graph_ir/rnn_declare.h new file mode 100644 index 00000000000..2bf2e03e80b --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/rnn_declare.h @@ -0,0 +1,54 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "ops/rnn.h" +#include "transform/graph_ir/op_declare/op_declare_macro.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现BasicLSTMCell的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BasicLSTMCell) +DECLARE_OP_USE_OUTPUT(BasicLSTMCell) +//对实现BasicLSTMCellInputGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BasicLSTMCellInputGrad) +DECLARE_OP_USE_OUTPUT(BasicLSTMCellInputGrad) +//对实现BasicLSTMCellWeightGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BasicLSTMCellWeightGrad) +DECLARE_OP_USE_OUTPUT(BasicLSTMCellWeightGrad) +//对实现BasicLSTMCellCStateGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BasicLSTMCellCStateGrad) +DECLARE_OP_USE_OUTPUT(BasicLSTMCellCStateGrad) +//对实现LSTMInputGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(LSTMInputGrad) +DECLARE_OP_USE_OUTPUT(LSTMInputGrad) +//对实现DynamicRNN的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DynamicRNN) +DECLARE_OP_USE_OUTPUT(DynamicRNN) +//对实现DynamicRNNGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DynamicRNNGrad) +DECLARE_OP_USE_OUTPUT(DynamicRNNGrad) +//对实现DynamicGRUV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DynamicGRUV2) +DECLARE_OP_USE_OUTPUT(DynamicGRUV2) +//对实现DynamicGRUV2Grad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DynamicGRUV2Grad) +DECLARE_OP_USE_OUTPUT(DynamicGRUV2Grad) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RNN_DECLARE_H_ -- 2.34.1 From 2027d3bbeb34aa4a59fa5d15d9417f6b13a2b8c9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:54:39 +0800 Subject: [PATCH 060/160] ADD file via upload --- .../transform/graph_ir/rpn_ops_declare.cc | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.cc new file mode 100644 index 00000000000..453ad36799d --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.cc @@ -0,0 +1,29 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/rpn_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// NMSWithMask +INPUT_MAP(NMSWithMask) = {{1, INPUT_DESC(box_scores)}};//将变量box_scores处理并存入对应InputDesc结构体的相应变量中，存入NMSWithMask对应空间内并用input_map指针保存 +ATTR_MAP(NMSWithMask) = {{"iou_threshold", ATTR_DESC(iou_threshold, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入NMSWithMask对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(NMSWithMask) = { + {0, OUTPUT_DESC(selected_boxes)}, {1, OUTPUT_DESC(selected_idx)}, {2, OUTPUT_DESC(selected_mask)}}; +//将变量selected_boxes、selected_idx、selected_mask处理并存入对应OutputDesc结构体的相应变量中，存入NMSWithMask对应空间内并用output_map_指针保存 +REG_ADPT_DESC(NMSWithMask, kNameNMSWithMask, ADPT_DESC(NMSWithMask))//构造指向NMSWithMask的指针并储存，创建结构体RegAdptDescNMSWithMask +} // namespace mindspore::transform -- 2.34.1 From 3df9565af51d791c5b53054dfbc557931bed3681 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:55:34 +0800 Subject: [PATCH 061/160] ADD file via upload --- .../transform/graph_ir/rpn_ops_declare.h | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.h new file mode 100644 index 00000000000..9a5ae4c3084 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/rpn_ops_declare.h @@ -0,0 +1,30 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/rpn_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现NMSWithMask的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(NMSWithMask) +DECLARE_OP_USE_OUTPUT(NMSWithMask) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_RPN_OPS_DECLARE_H_ -- 2.34.1 From 86d4a90bd1b19a9d86220e23497254c674677604 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:55:51 +0800 Subject: [PATCH 062/160] ADD file via upload --- .../graph_ir/selection_ops_declare.cc | 289 ++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.cc new file mode 100644 index 00000000000..e6f79ad5a9b --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.cc @@ -0,0 +1,289 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include //提供vector数组构建函数模版等 +#include "transform/graph_ir/op_declare/selection_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// CumsumD +INPUT_MAP(CumsumD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入CumsumD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(CumsumD) = {{2, ATTR_DESC(axis, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入CumsumD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(CumsumD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits())}, + {"reverse", ATTR_DESC(reverse, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入CumsumD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(CumsumD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入CumsumD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(CumsumD, kNameCumSum, ADPT_DESC(CumsumD))//构造指向CumsumD的指针并储存，创建结构体RegAdptDescCumsumD + +// GatherV2 +INPUT_MAP(GatherV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(axis)}}; +//将变量x、indices、axis处理并存入对应InputDesc结构体的相应变量中，存入GatherV2对应空间内并用input_map指针保存 +ATTR_MAP(GatherV2) = EMPTY_ATTR_MAP;//将空变量存入GatherV2对应空间并用attr_map_指针保存 +OUTPUT_MAP(GatherV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherV2对应空间内并用output_map_指针保存 + +// CumprodD +INPUT_MAP(CumprodD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入CumprodD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(CumprodD) = {{2, ATTR_DESC(axis, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入CumprodD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(CumprodD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits())}, + {"reverse", ATTR_DESC(reverse, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入CumprodD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(CumprodD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入CumprodD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(CumprodD, kNameCumProd, ADPT_DESC(CumprodD))//构造指向CumprodD的指针并储存，创建结构体RegAdptDescCumprodD + +//SliceD +INPUT_MAP(SliceD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SliceD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(SliceD) = {{2, ATTR_DESC(offsets, AnyTraits(), AnyTraits>())}, + {3, ATTR_DESC(size, AnyTraits(), AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SliceD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +ATTR_MAP(SliceD) = EMPTY_ATTR_MAP;//将空变量存入SliceD对应空间并用attr_map_指针保存 +OUTPUT_MAP(SliceD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入SliceD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SliceD, kNameSlice, ADPT_DESC(SliceD))//构造指向SliceD的指针并储存，创建结构体RegAdptDescSliceD + +// TopK +INPUT_MAP(TopK) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(k)}}; +//将变量x、k处理并存入对应InputDesc结构体的相应变量中，存入TopK对应空间内并用input_map指针保存 +ATTR_MAP(TopK) = {{"sorted", ATTR_DESC(sorted, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入TopK对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(TopK) = {{0, OUTPUT_DESC(values)}, {1, OUTPUT_DESC(indices)}}; +//将变量values、indices处理并存入对应OutputDesc结构体的相应变量中，存入TopK对应空间内并用output_map_指针保存 +REG_ADPT_DESC(TopK, kNameTopK, ADPT_DESC(TopK))//构造指向TopK的指针并储存，创建结构体RegAdptDescTopK + +// InTopK +INPUT_MAP(InTopKD) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}};//将变量x1、x2处理并存入对应InputDesc结构体的相应变量中，存入InTopKD对应空间内并用input_map指针保存 +ATTR_MAP(InTopKD) = {{"k", ATTR_DESC(k, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入InTopKD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(InTopKD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InTopKD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(InTopKD, kNameInTopKD, ADPT_DESC(InTopKD))//构造指向InTopKD的指针并储存，创建结构体RegAdptDescInTopKD + +// TileD +INPUT_MAP(TileD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入TileD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(TileD) = {{2, ATTR_DESC(multiples, AnyTraits(), AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SliceD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +ATTR_MAP(TileD) = EMPTY_ATTR_MAP;//将空变量存入TileD对应空间并用attr_map_指针保存 +OUTPUT_MAP(TileD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入TileD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(TileD, kNameTile, ADPT_DESC(TileD))//构造指向TileD的指针并储存，创建结构体RegAdptDescTileD + +// OneHot +INPUT_MAP(OneHot) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(depth)}, {3, INPUT_DESC(on_value)}, {4, INPUT_DESC(off_value)}}; +//将变量x、depth、on_value、off_value处理并存入对应InputDesc结构体的相应变量中，存入OneHot对应空间内并用input_map指针保存 +ATTR_MAP(OneHot) = {{"axis", ATTR_DESC(axis, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入OneHot对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(OneHot) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入OneHot对应空间内并用output_map_指针保存 +REG_ADPT_DESC(OneHot, prim::kPrimOneHot->name(), ADPT_DESC(OneHot))//构造指向OneHot的指针并储存，创建结构体RegAdptDescOneHot + +// GatherV2D +INPUT_MAP(GatherV2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}}; +//将变量x、indices处理并存入对应InputDesc结构体的相应变量中，存入GatherV2D对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(GatherV2D) = {{3, ATTR_DESC(axis, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入GatherV2D对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(GatherV2D) = EMPTY_ATTR_MAP;//将空变量存入GatherV2D对应空间并用attr_map_指针保存 +OUTPUT_MAP(GatherV2D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherV2D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(GatherV2D, prim::kPrimGather->name(), ADPT_DESC(GatherV2D))//构造指向GatherV2D的指针并储存，创建结构体RegAdptDescGatherV2D +REG_ADPT_DESC(Gather, kNameGather, ADPT_DESC(GatherV2D))//构造指向Gather的指针并储存，创建结构体RegAdptDescGather + +// ScatterNdD +INPUT_MAP(ScatterNdD) = {{1, INPUT_DESC(indices)}, {2, INPUT_DESC(x)}}; +//将变量indices、x处理并存入对应InputDesc结构体的相应变量中，存入ScatterNdD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(ScatterNdD) = { + {3, ATTR_DESC(shape, AnyTraits>(), AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ScatterNdD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +ATTR_MAP(ScatterNdD) = EMPTY_ATTR_MAP;//将空变量存入ScatterNdD对应空间并用attr_map_指针保存 +OUTPUT_MAP(ScatterNdD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ScatterNdD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ScatterNdD, kNameScatterNdD, ADPT_DESC(ScatterNdD))//构造指向ScatterNdD的指针并储存，创建结构体RegAdptDescScatterNdD + +// ScatterNonAliasingAdd +INPUT_MAP(ScatterNonAliasingAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +//将变量x、indices、updates处理并存入对应InputDesc结构体的相应变量中，存入ScatterNonAliasingAdd对应空间内并用input_map指针保存 +ATTR_MAP(ScatterNonAliasingAdd) = EMPTY_ATTR_MAP;//将空变量存入ScatterNonAliasingAdd对应空间并用attr_map_指针保存 +OUTPUT_MAP(ScatterNonAliasingAdd) = {{0, OUTPUT_DESC(y)}}; +//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ScatterNonAliasingAdd对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ScatterNonAliasingAdd, kNameScatterNonAliasingAdd, ADPT_DESC(ScatterNonAliasingAdd)) +//构造指向ScatterNonAliasingAdd的指针并储存，创建结构体RegAdptDescScatterNonAliasingAdd + +// GatherNd +INPUT_MAP(GatherNd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}}; +//将变量x、indices处理并存入对应InputDesc结构体的相应变量中，存入GatherNd对应空间内并用input_map指针保存 +ATTR_MAP(GatherNd) = EMPTY_ATTR_MAP;//将空变量存入GatherNd对应空间并用attr_map_指针保存 +OUTPUT_MAP(GatherNd) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherNd对应空间内并用output_map_指针保存 +REG_ADPT_DESC(GatherNd, kNameGatherNd, ADPT_DESC(GatherNd))//构造指向GatherNd的指针并储存，创建结构体RegAdptDescGatherNd + +// GatherD +INPUT_MAP(GatherD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(dim)}, {3, INPUT_DESC(index)}}; +//将变量x、dim、index处理并存入对应InputDesc结构体的相应变量中，存入GatherD对应空间内并用input_map指针保存 +ATTR_MAP(GatherD) = EMPTY_ATTR_MAP;//将空变量存入GatherD对应空间并用attr_map_指针保存 +OUTPUT_MAP(GatherD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入GatherD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(GatherD, kNameGatherD, ADPT_DESC(GatherD))//构造指向GatherD的指针并储存，创建结构体RegAdptDescGatherD + +// Range +INPUT_MAP(RangeD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入RangeD对应空间内并用input_map指针保存 +ATTR_MAP(RangeD) = {{"start", ATTR_DESC(start, AnyTraits())}, + {"limit", ATTR_DESC(limit, AnyTraits())}, + {"delta", ATTR_DESC(delta, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入RangeD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(RangeD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入RangeD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(RangeD, kNameRange, ADPT_DESC(RangeD))//构造指向RangeD的指针并储存，创建结构体RegAdptDescRangeD + +// InplaceAddD +INPUT_MAP(InplaceAddD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}}; +//将变量x、v处理并存入对应InputDesc结构体的相应变量中，存入InplaceAddD对应空间内并用input_map指针保存 +ATTR_MAP(InplaceAddD) = {{"indices", ATTR_DESC(indices, AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入InplaceAddD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(InplaceAddD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceAddD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(InplaceAddD, kNameInplaceAddD, ADPT_DESC(InplaceAddD))//构造指向InplaceAddD的指针并储存，创建结构体RegAdptDescInplaceAddD + +// InplaceSubD +INPUT_MAP(InplaceSubD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}}; +//将变量x、v处理并存入对应InputDesc结构体的相应变量中，存入InplaceSubD对应空间内并用input_map指针保存 +ATTR_MAP(InplaceSubD) = {{"indices", ATTR_DESC(indices, AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入InplaceSubD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(InplaceSubD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceSubD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(InplaceSubD, kNameInplaceSubD, ADPT_DESC(InplaceSubD))//构造指向InplaceSubD的指针并储存，创建结构体RegAdptDescInplaceSubD + +// InplaceUpdateD +INPUT_MAP(InplaceUpdateD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}}; +//将变量x、v处理并存入对应InputDesc结构体的相应变量中，存入InplaceUpdateD对应空间内并用input_map指针保存 +ATTR_MAP(InplaceUpdateD) = {{"indices", ATTR_DESC(indices, AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入InplaceUpdateD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(InplaceUpdateD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceUpdateD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(InplaceUpdateD, kNameInplaceUpdateD, ADPT_DESC(InplaceUpdateD))//构造指向InplaceUpdateD的指针并储存，创建结构体RegAdptDescInplaceUpdateD + +// Select +INPUT_MAP(Select) = {{1, INPUT_DESC(condition)}, {2, INPUT_DESC(x1)}, {3, INPUT_DESC(x2)}}; +//将变量condition、x1、x2处理并存入对应InputDesc结构体的相应变量中，存入Select对应空间内并用input_map指针保存 +ATTR_MAP(Select) = EMPTY_ATTR_MAP;//将空变量存入Select对应空间并用attr_map_指针保存 +OUTPUT_MAP(Select) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入InplaceUpdateD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Select, prim::kPrimSelect->name(), ADPT_DESC(Select))//构造指向Select的指针并储存，创建结构体RegAdptDescSelect + +// StridedSliceGrad +INPUT_MAP(StridedSliceGrad) = { + {1, INPUT_DESC(dy)}, {2, INPUT_DESC(shape)}, {3, INPUT_DESC(begin)}, {4, INPUT_DESC(end)}, {5, INPUT_DESC(strides)}}; +//将变量dy、shape、begin、end、strides处理并存入对应InputDesc结构体的相应变量中，存入StridedSliceGrad对应空间内并用input_map指针保存 +ATTR_MAP(StridedSliceGrad) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入StridedSliceGrad对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(StridedSliceGrad) = {{0, OUTPUT_DESC(output)}}; +//将变量output处理并存入对应OutputDesc结构体的相应变量中，存入StridedSliceGrad对应空间内并用output_map_指针保存 +REG_ADPT_DESC(StridedSliceGrad, kNameStridedSliceGrad, ADPT_DESC(StridedSliceGrad)) +//构造指向StridedSliceGrad的指针并储存，创建结构体RegAdptDescStridedSliceGrad + +// StridedSlice +INPUT_MAP(StridedSlice) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(strides)}}; +//将变量x、begin、end、strides处理并存入对应InputDesc结构体的相应变量中，存入StridedSlice对应空间内并用input_map指针保存 +ATTR_MAP(StridedSlice) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入StridedSlice对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(StridedSlice) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入StridedSlice对应空间内并用output_map_指针保存 +REG_ADPT_DESC(StridedSlice, kNameStridedSlice, ADPT_DESC(StridedSlice)) +//构造指向StridedSlice的指针并储存，创建结构体RegAdptDescStridedSlice + +// StridedSliceV2 +INPUT_MAP(StridedSliceV2) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(axes)}, {5, INPUT_DESC(strides)}}; +//将变量x、begin、end、axes、strides处理并存入对应InputDesc结构体的相应变量中，存入StridedSlice对应空间内并用input_map指针保存 +ATTR_MAP(StridedSliceV2) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入StridedSliceV2对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(StridedSliceV2) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入StridedSliceV2对应空间内并用output_map_指针保存 +REG_ADPT_DESC(StridedSliceV2, kNameStridedSliceV2, ADPT_DESC(StridedSliceV2)) +//构造指向StridedSliceV2的指针并储存，创建结构体RegAdptDescStridedSliceV2 + +// UnsortedSegmentSum +INPUT_MAP(UnsortedSegmentSumD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +//将变量x、segment_ids处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentSumD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(UnsortedSegmentSumD) = {{3, ATTR_DESC(num_segments, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入UnsortedSegmentSumD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(UnsortedSegmentSumD) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentSumD对应空间并用attr_map_指针保存 +OUTPUT_MAP(UnsortedSegmentSumD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentSumD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(UnsortedSegmentSumD, prim::kPrimUnsortedSegmentSum->name(), ADPT_DESC(UnsortedSegmentSumD)) +//构造指向UnsortedSegmentSumD的指针并储存，创建结构体RegAdptDescUnsortedSegmentSumD + +// UnsortedSegmentProdD +INPUT_MAP(UnsortedSegmentProdD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +//将变量x、segment_ids处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentProdD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(UnsortedSegmentProdD) = {{3, ATTR_DESC(num_segments, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入UnsortedSegmentProdD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(UnsortedSegmentProdD) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentProdD对应空间并用attr_map_指针保存 +OUTPUT_MAP(UnsortedSegmentProdD) = {{0, OUTPUT_DESC(y)}}; +//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentProdD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(UnsortedSegmentProdD, kNameUnsortedSegmentProdD, ADPT_DESC(UnsortedSegmentProdD)) +//构造指向UnsortedSegmentProdD的指针并储存，创建结构体RegAdptDescUnsortedSegmentProdD + +// UnsortedSegmentMaxD +INPUT_MAP(UnsortedSegmentMaxD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +//将变量x、segment_ids处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentMaxD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(UnsortedSegmentMaxD) = {{3, ATTR_DESC(num_segments, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入UnsortedSegmentMaxD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +ATTR_MAP(UnsortedSegmentMaxD) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentMaxD对应空间并用attr_map_指针保存 +OUTPUT_MAP(UnsortedSegmentMaxD) = {{0, OUTPUT_DESC(y)}}; +//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentMaxD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(UnsortedSegmentMaxD, kNameUnsortedSegmentMaxD, ADPT_DESC(UnsortedSegmentMaxD)) +//构造指向UnsortedSegmentMaxD的指针并储存，创建结构体RegAdptDescUnsortedSegmentMaxD + +// UnsortedSegmentMin +INPUT_MAP(UnsortedSegmentMin) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}, {3, INPUT_DESC(num_segments)}}; +//将变量x、segment_ids、num_segments处理并存入对应InputDesc结构体的相应变量中，存入UnsortedSegmentMin对应空间内并用input_map指针保存 +ATTR_MAP(UnsortedSegmentMin) = EMPTY_ATTR_MAP;//将空变量存入UnsortedSegmentMin对应空间并用attr_map_指针保存 +OUTPUT_MAP(UnsortedSegmentMin) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入UnsortedSegmentMin对应空间内并用output_map_指针保存 +REG_ADPT_DESC(UnsortedSegmentMin, prim::kPrimUnsortedSegmentMin->name(), ADPT_DESC(UnsortedSegmentMin)) +//构造指向UnsortedSegmentMin的指针并储存，创建结构体RegAdptDescUnsortedSegmentMin + +// ReverseV2 +INPUT_MAP(ReverseV2D) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ReverseV2D对应空间内并用input_map指针保存 +ATTR_MAP(ReverseV2D) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ReverseV2D对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(ReverseV2D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ReverseV2D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ReverseV2D, kNameReverseV2, ADPT_DESC(ReverseV2D)) +//构造指向ReverseV2D的指针并储存，创建结构体RegAdptDescReverseV2D +} // namespace mindspore::transform -- 2.34.1 From 4021fe0d548a918d92f4c2d3b6762788562e99e7 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:56:06 +0800 Subject: [PATCH 063/160] ADD file via upload --- .../graph_ir/selection_ops_declare.h | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.h new file mode 100644 index 00000000000..8d3b0479878 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/selection_ops_declare.h @@ -0,0 +1,114 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/selection_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现SliceD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SliceD) +DECLARE_OP_USE_INPUT_ATTR(SliceD) +DECLARE_OP_USE_OUTPUT(SliceD) +//对实现ScatterNdD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ScatterNdD) +DECLARE_OP_USE_INPUT_ATTR(ScatterNdD) +DECLARE_OP_USE_OUTPUT(ScatterNdD) +//对实现ScatterNonAliasingAdd的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ScatterNonAliasingAdd) +DECLARE_OP_USE_OUTPUT(ScatterNonAliasingAdd) +//对实现GatherNd的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(GatherNd) +DECLARE_OP_USE_OUTPUT(GatherNd) +//对实现GatherD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(GatherD) +DECLARE_OP_USE_OUTPUT(GatherD) +//对实现TopK的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(TopK) +DECLARE_OP_USE_OUTPUT(TopK) +//对实现InTopKD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(InTopKD) +DECLARE_OP_USE_OUTPUT(InTopKD) +//对实现Select的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Select) +DECLARE_OP_USE_OUTPUT(Select) +//对实现StridedSliceGrad的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(StridedSliceGrad) +DECLARE_OP_USE_OUTPUT(StridedSliceGrad) +//对实现StridedSlice的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(StridedSlice) +DECLARE_OP_USE_OUTPUT(StridedSlice) +//对实现StridedSliceV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(StridedSliceV2) +DECLARE_OP_USE_OUTPUT(StridedSliceV2) +//对实现UnsortedSegmentSumD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(UnsortedSegmentSumD) +DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentSumD) +DECLARE_OP_USE_OUTPUT(UnsortedSegmentSumD) +//对实现UnsortedSegmentProdD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(UnsortedSegmentProdD) +DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentProdD) +DECLARE_OP_USE_OUTPUT(UnsortedSegmentProdD) +//对实现UnsortedSegmentMaxD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(UnsortedSegmentMaxD) +DECLARE_OP_USE_INPUT_ATTR(UnsortedSegmentMaxD) +DECLARE_OP_USE_OUTPUT(UnsortedSegmentMaxD) +//对实现UnsortedSegmentMin的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(UnsortedSegmentMin) +DECLARE_OP_USE_OUTPUT(UnsortedSegmentMin) +//对实现CumprodD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(CumprodD) +DECLARE_OP_USE_INPUT_ATTR(CumprodD) +DECLARE_OP_USE_OUTPUT(CumprodD) +//对实现TileD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(TileD) +DECLARE_OP_USE_INPUT_ATTR(TileD) +DECLARE_OP_USE_OUTPUT(TileD) +//对实现OneHot的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(OneHot) +DECLARE_OP_USE_OUTPUT(OneHot) +//对实现GatherV2D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(GatherV2D) +DECLARE_OP_USE_INPUT_ATTR(GatherV2D) +DECLARE_OP_USE_OUTPUT(GatherV2D) +//对实现RangeD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(RangeD) +DECLARE_OP_USE_OUTPUT(RangeD) +//对实现InplaceAddD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(InplaceAddD) +DECLARE_OP_USE_OUTPUT(InplaceAddD) +//对实现InplaceSubD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(InplaceSubD) +DECLARE_OP_USE_OUTPUT(InplaceSubD) +//对实现InplaceUpdateD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(InplaceUpdateD) +DECLARE_OP_USE_OUTPUT(InplaceUpdateD) +//对实现CumsumD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(CumsumD) +DECLARE_OP_USE_INPUT_ATTR(CumsumD) +DECLARE_OP_USE_OUTPUT(CumsumD) +//对实现GatherV2的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(GatherV2) +DECLARE_OP_USE_OUTPUT(GatherV2) +//对实现ReverseV2D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ReverseV2D) +DECLARE_OP_USE_OUTPUT(ReverseV2D) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SELECTION_OPS_DECLARE_H_ -- 2.34.1 From 5e1564a160cab73d1a9926d4d681a27787f1e8da Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:56:22 +0800 Subject: [PATCH 064/160] ADD file via upload --- .../graph_ir/split_combination_ops_declare.cc | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.cc new file mode 100644 index 00000000000..2911b7c5f25 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.cc @@ -0,0 +1,79 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/split_combination_ops_declare.h"//按照路径寻找以下文件，导入到本文件 +#include //提供vector数组构建函数模版等 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// SplitD +INPUT_MAP(SplitD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SplitD对应空间内并用input_map指针保存 +ATTR_MAP(SplitD) = {{"axis", ATTR_DESC(split_dim, AnyTraits())}, + {"output_num", ATTR_DESC(num_split, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SplitD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +DYN_OUTPUT_MAP(SplitD) = {{0, DYN_OUTPUT_DESC(y)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入SplitD对应空间内并用input_map指针保存 +REG_ADPT_DESC(SplitD, kNameSplitD, ADPT_DESC(SplitD))//构造指向SplitD的指针并储存，创建结构体RegAdptDescSplitD + +// Pack +INPUT_MAP(Pack) = EMPTY_INPUT_MAP;//将空变量存入Pack对应空间并用input_map指针保存 +DYN_INPUT_MAP(Pack) = {{1, DYN_INPUT_DESC(x)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入Pack对应空间内并用dyn_input_map_指针保存 +ATTR_MAP(Pack) = {{"num", ATTR_DESC(N, AnyTraits())}, {"axis", ATTR_DESC(axis, AnyTraits())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入Pack对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(Pack) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Pack对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Pack, prim::kStack, ADPT_DESC(Pack))//构造指向Pack的指针并储存，创建结构体RegAdptDescPack + +// ParallelConcat +INPUT_MAP(ParallelConcat) = EMPTY_INPUT_MAP;//将空变量存入ParallelConcat对应空间并用input_map指针保存 +DYN_INPUT_MAP(ParallelConcat) = {{1, DYN_INPUT_DESC(values)}}; +//将变量values处理并存入对应DynOutputDesc结构体的相应变量中，存入ParallelConcat对应空间内并用dyn_input_map_指针保存 +ATTR_MAP(ParallelConcat) = { + {"shape", ATTR_DESC(shape, AnyTraits>())}, + {"N", ATTR_DESC(N, AnyTraits())}, +};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入ParallelConcat对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(ParallelConcat) = {{0, OUTPUT_DESC(output_data)}}; +//将变量output_data处理并存入对应OutputDesc结构体的相应变量中，存入ParallelConcat对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ParallelConcat, kNameParallelConcat, ADPT_DESC(ParallelConcat)) +//构造指向ParallelConcat的指针并储存，创建结构体RegAdptDescParallelConcat + +// ConcatD +INPUT_MAP(ConcatD) = EMPTY_INPUT_MAP;//将空变量存入ConcatD对应空间并用input_map指针保存 +DYN_INPUT_MAP(ConcatD) = {{1, DYN_INPUT_DESC(x)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入ConcatD对应空间内并用dyn_input_map_指针保存 +ATTR_MAP(ConcatD) = { + {"axis", ATTR_DESC(concat_dim, AnyTraits())}, + {"inputNums", ATTR_DESC(N, AnyTraits())}, +};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入ConcatD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ConcatD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ConcatD)对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ConcatD, prim::kPrimConcat->name(), ADPT_DESC(ConcatD))//构造指向ConcatD的指针并储存，创建结构体RegAdptDescConcatD + +// ConcatV2D Inference for tf +INPUT_MAP(ConcatV2D) = EMPTY_INPUT_MAP;//将空变量存入ConcatV2D对应空间并用input_map指针保存 +DYN_INPUT_MAP(ConcatV2D) = {{1, DYN_INPUT_DESC(x)}};//将变量x处理并存入对应DynOutputDesc结构体的相应变量中，存入ConcatV2D对应空间内并用dyn_input_map_指针保存 +ATTR_MAP(ConcatV2D) = { + {"axis", ATTR_DESC(concat_dim, AnyTraits())}, + {"N", ATTR_DESC(N, AnyTraits())}, +};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入ConcaV2tD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(ConcatV2D) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ConcatV2D对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ConcatV2D, kNameConcatV2D, ADPT_DESC(ConcatV2D))//构造指向ConcatV2D的指针并储存，创建结构体RegAdptDescConcatV2D +} // namespace mindspore::transform -- 2.34.1 From f82b196e027116c65283bd661188eb2a0bc3307c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:56:40 +0800 Subject: [PATCH 065/160] ADD file via upload --- .../graph_ir/split_combination_ops_declare.h | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.h new file mode 100644 index 00000000000..730aaf21788 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/split_combination_ops_declare.h @@ -0,0 +1,47 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SPLIT_COMBINATION_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SPLIT_COMBINATION_OPS_DECLARE_H_ +//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SPLIT_COMBINATION_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/split_combination_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现SplitD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SplitD) +DECLARE_OP_USE_DYN_OUTPUT(SplitD) +//对实现ConcatD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ConcatD) +DECLARE_OP_USE_DYN_INPUT(ConcatD) +DECLARE_OP_USE_OUTPUT(ConcatD) +//对实现ConcatV2D的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ConcatV2D) +DECLARE_OP_USE_DYN_INPUT(ConcatV2D) +DECLARE_OP_USE_OUTPUT(ConcatV2D) +//对实现ParallelConcat的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ParallelConcat) +DECLARE_OP_USE_DYN_INPUT(ParallelConcat) +DECLARE_OP_USE_OUTPUT(ParallelConcat) +//对实现Pack的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Pack) +DECLARE_OP_USE_DYN_INPUT(Pack) +DECLARE_OP_USE_OUTPUT(Pack) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_SPLIT_COMBINATION_OPS_DECLARE_H_ -- 2.34.1 From 5f58a5a0d5c56404bf5e3e211de36a107b4efa14 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:57:05 +0800 Subject: [PATCH 066/160] ADD file via upload --- .../transform/graph_ir/state_ops_declare.cc | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.cc new file mode 100644 index 00000000000..51f77af4324 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.cc @@ -0,0 +1,23 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/state_ops_declare.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// Variable +INPUT_MAP(Variable) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Variable对应空间内并用input_map指针保存 +ATTR_MAP(Variable) = EMPTY_ATTR_MAP;//将空变量存入Variable对应空间并用attr_map_指针保存 +} // namespace mindspore::transform -- 2.34.1 From dedd7cf4daa6e966fa2b60a86700318a726f0b0e Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:57:23 +0800 Subject: [PATCH 067/160] ADD file via upload --- .../transform/graph_ir/state_ops_declare.h | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.h new file mode 100644 index 00000000000..33f04c873ab --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/state_ops_declare.h @@ -0,0 +1,29 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_STATE_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_STATE_OPS_DECLARE_H_//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_STATE_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/state_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现BasicLSTMCell的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Variable) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_STATE_OPS_DECLARE_H_ -- 2.34.1 From 0c302ec78620c3692ff60e4d794dfa024d6682e2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:57:41 +0800 Subject: [PATCH 068/160] ADD file via upload --- .../graph_ir/transformation_ops_declare.cc | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.cc b/mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.cc new file mode 100644 index 00000000000..8969bdb5c32 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.cc @@ -0,0 +1,124 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/transformation_ops_declare.h"//按照路径寻找以下文件，导入到本文件 +#include //提供vector数组构建函数模版等 + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +// Flatten +INPUT_MAP(Flatten) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Flatten对应空间内并用input_map指针保存 +ATTR_MAP(Flatten) = EMPTY_ATTR_MAP;//将空变量存入Flatten对应空间并用attr_map_指针保存 +OUTPUT_MAP(Flatten) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入Flatten对应空间内并用output_map_指针保存 +REG_ADPT_DESC(Flatten, prim::kPrimFlatten->name(), ADPT_DESC(Flatten))//构造指向Flatten的指针并储存，创建结构体RegAdptDescFlatten + +// Unpack +INPUT_MAP(Unpack) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入Unpack对应空间内并用input_map指针保存 +ATTR_MAP(Unpack) = {{"axis", ATTR_DESC(axis, AnyTraits())}, {"num", ATTR_DESC(num, AnyTraits())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入Unpack对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +DYN_OUTPUT_MAP(Unpack) = {{0, DYN_OUTPUT_DESC(y)}};//将变量y处理并存入对应DynOutputDesc结构体的相应变量中，存入Unpack对应空间内并用input_map指针保存 +REG_ADPT_DESC(Unpack, prim::kUnstack, ADPT_DESC(Unpack))//构造指向Unpack的指针并储存，创建结构体RegAdptDescUnpack + +// ExtractImagePatches +INPUT_MAP(ExtractImagePatches) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入ExtractImagePatches对应空间内并用input_map指针保存 +ATTR_MAP(ExtractImagePatches) = { + {"ksizes", ATTR_DESC(ksizes, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"rates", ATTR_DESC(rates, AnyTraits(), AnyTraits>())}, + {"padding", ATTR_DESC(padding, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入ParallelConcat对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(ExtractImagePatches) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入ExtractImagePatches对应空间内并用output_map_指针保存 +REG_ADPT_DESC(ExtractImagePatches, kNameExtractImagePatches, ADPT_DESC(ExtractImagePatches)) +//构造指向ExtractImagePatches的指针并储存，创建结构体RegAdptDescExtractImagePatches + +// Transpose +INPUT_MAP(TransposeD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入TransposeD对应空间内并用input_map指针保存 +INPUT_ATTR_MAP(TransposeD) = {{2, ATTR_DESC(perm, AnyTraits(), AnyTraits>())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入TransposeD对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +// std:vector<>的作用为构建<>内类型的容量可变的数组 +ATTR_MAP(TransposeD) = EMPTY_ATTR_MAP;//将空变量存入TransposeD对应空间并用attr_map_指针保存 +// Do not set Transpose operator output descriptor不要在输出描述符中设置转置运算符 +REG_ADPT_DESC(TransposeD, prim::kPrimTranspose->name(), ADPT_DESC(TransposeD)) +//构造指向TransposeD的指针并储存，创建结构体RegAdptDescTransposeD + +// SpaceToDepth +INPUT_MAP(SpaceToDepth) = {{1, INPUT_DESC(x)}};;//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SpaceToDepth对应空间内并用input_map指针保存 +ATTR_MAP(SpaceToDepth) = {{"block_size", ATTR_DESC(block_size, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入SpaceToDepth对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(SpaceToDepth) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入SpaceToDepth对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SpaceToDepth, kNameSpaceToDepth, ADPT_DESC(SpaceToDepth))//构造指向SpaceToDepth的指针并储存，创建结构体RegAdptDescSpaceToDepth + +// DepthToSpace +INPUT_MAP(DepthToSpace) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入DepthToSpace对应空间内并用input_map指针保存 +ATTR_MAP(DepthToSpace) = {{"block_size", ATTR_DESC(block_size, AnyTraits())}};//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +// 存入DepthToSpace对应空间并用attr_map_指针保存 +// 其中AnyTraits<>的作用为将<>内类型进行构建 +OUTPUT_MAP(DepthToSpace) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入DepthToSpace对应空间内并用output_map_指针保存 +REG_ADPT_DESC(DepthToSpace, kNameDepthToSpace, ADPT_DESC(DepthToSpace))//构造指向DepthToSpace的指针并储存，创建结构体RegAdptDescDepthToSpace + +// SpaceToBatchD +INPUT_MAP(SpaceToBatchD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SpaceToBatchD对应空间内并用input_map指针保存 +ATTR_MAP(SpaceToBatchD) = { + {"block_size", ATTR_DESC(block_size, AnyTraits())}, + {"paddings", ATTR_DESC(paddings, AnyTraits>>(), AnyTraits>())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入SpaceToBatchD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(SpaceToBatchD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入SpaceToBatchD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SpaceToBatchD, kNameSpaceToBatch, ADPT_DESC(SpaceToBatchD))//构造指向SpaceToBatchD的指针并储存，创建结构体RegAdptDescSpaceToBatchD + +// SpaceToBatchNDD +INPUT_MAP(SpaceToBatchNDD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入SpaceToBatchNDD对应空间内并用input_map指针保存 +ATTR_MAP(SpaceToBatchNDD) = { + {"block_shape", ATTR_DESC(block_shape, AnyTraits>())}, + {"paddings", ATTR_DESC(paddings, AnyTraits>>(), AnyTraits>())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入SpaceToBatchNDD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(SpaceToBatchNDD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入SpaceToBatchNDD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(SpaceToBatchNDD, kNameSpaceToBatchNDD, ADPT_DESC(SpaceToBatchNDD))//构造指向SpaceToBatchNDD的指针并储存，创建结构体RegAdptDescSpaceToBatchNDD + +// BatchToSpaceD +INPUT_MAP(BatchToSpaceD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入BatchToSpaceD对应空间内并用input_map指针保存 +ATTR_MAP(BatchToSpaceD) = { + {"block_size", ATTR_DESC(block_size, AnyTraits())}, + {"crops", ATTR_DESC(crops, AnyTraits>>(), AnyTraits>())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入BatchToSpaceD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(BatchToSpaceD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入BatchToSpaceD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BatchToSpaceD, kNameBatchToSpace, ADPT_DESC(BatchToSpaceD))//构造指向BatchToSpaceD的指针并储存，创建结构体RegAdptDescBatchToSpaceD + +// BatchToSpaceNDD +INPUT_MAP(BatchToSpaceNDD) = {{1, INPUT_DESC(x)}};//将变量x处理并存入对应InputDesc结构体的相应变量中，存入BatchToSpaceNDD对应空间内并用input_map指针保存 +ATTR_MAP(BatchToSpaceNDD) = { + {"block_shape", ATTR_DESC(block_shape, AnyTraits>())}, + {"crops", ATTR_DESC(crops, AnyTraits>>(), AnyTraits>())}}; +//对相应变量处理并存入对应AttrDesc结构体的相应变量中 +//存入BatchToSpaceNDD对应空间并用attr_map_指针保存 +//其中AnyTraits<>的作用为将<>内类型进行构建 +//std:vector<>的作用为构建<>内类型的容量可变的数组 +OUTPUT_MAP(BatchToSpaceNDD) = {{0, OUTPUT_DESC(y)}};//将变量y处理并存入对应OutputDesc结构体的相应变量中，存入BatchToSpaceNDD对应空间内并用output_map_指针保存 +REG_ADPT_DESC(BatchToSpaceNDD, kNameBatchToSpaceNd, ADPT_DESC(BatchToSpaceNDD))//构造指向BatchToSpaceNDD的指针并储存，创建结构体RegAdptDescBatchToSpaceNDD +} // namespace mindspore::transform -- 2.34.1 From d7f977a80df13246ea7aa8acd9a806198896593f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Thu, 29 Sep 2022 20:57:57 +0800 Subject: [PATCH 069/160] ADD file via upload --- .../graph_ir/transformation_ops_declare.h | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.h b/mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.h new file mode 100644 index 00000000000..cc92e0d1f20 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/transformation_ops_declare.h @@ -0,0 +1,58 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_TRANSFORMATION_OPS_DECLARE_H_//判断宏是否被定义，如果宏没有定义，则编译下面代码 +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_TRANSFORMATION_OPS_DECLARE_H_ +//定义预处理宏_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_TRANSFORMATION_OPS_DECLARE_H_ + +#include //导入标准库中的字符串类及相关操作 +#include "utils/hash_map.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "transform/graph_ir/op_declare/op_declare_macro.h" +#include "ops/transformation_ops.h" + +namespace mindspore::transform {//创建名为transform的空间，其空间处于空间mindspore下 +//对实现ExtractImagePatches的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(ExtractImagePatches) +DECLARE_OP_USE_OUTPUT(ExtractImagePatches) +//对实现Unpack的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Unpack) +DECLARE_OP_USE_DYN_OUTPUT(Unpack) +//对实现TransposeD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(TransposeD) +DECLARE_OP_USE_INPUT_ATTR(TransposeD) +//对实现Flatten的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(Flatten) +DECLARE_OP_USE_OUTPUT(Flatten) +//对实现SpaceToDepth的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SpaceToDepth) +DECLARE_OP_USE_OUTPUT(SpaceToDepth) +//对实现DepthToSpace的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(DepthToSpace) +DECLARE_OP_USE_OUTPUT(DepthToSpace) +//对实现SpaceToBatchD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SpaceToBatchD) +DECLARE_OP_USE_OUTPUT(SpaceToBatchD) +//对实现SpaceToBatchNDD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(SpaceToBatchNDD) +DECLARE_OP_USE_OUTPUT(SpaceToBatchNDD) +//对实现BatchToSpaceD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BatchToSpaceD) +DECLARE_OP_USE_OUTPUT(BatchToSpaceD) +//对实现BatchToSpaceNDD的对象转换后转移出相应存储空间 +DECLARE_OP_ADAPTER(BatchToSpaceNDD) +DECLARE_OP_USE_OUTPUT(BatchToSpaceNDD) +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_TRANSFORMATION_OPS_DECLARE_H_ -- 2.34.1 From a1612e8cee851d32f832c05a00ebaac850526ee0 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sat, 1 Oct 2022 19:29:39 +0800 Subject: [PATCH 070/160] ADD file via upload --- .../transform/graph_ir/mindir_exporter.cc | 1228 +++++++++++++++++ 1 file changed, 1228 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/mindir_exporter.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/mindir_exporter.cc b/mindspore2022/ccsrc/transform/graph_ir/mindir_exporter.cc new file mode 100644 index 00000000000..b2d87891b88 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/mindir_exporter.cc @@ -0,0 +1,1228 @@ +/** + * Copyright 2020-2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include //STL映射容器 +#include //空间配置器 +#include //STL通用模板类 +#include //STL通用算法 +#include //STL定义运算函数（代替运算符） +#include //STL中对文件操作的合集 + +#include "utils/hash_map.h" //在当前路径下引入头文件，下面同理 +#include "ir/tensor.h" +#include "ir/param_info.h" +#include "ir/func_graph.h" +#include "base/core_ops.h" +#include "proto/mind_ir.pb.h" +#include "utils/check_convert_utils.h" +#include "include/common/debug/dump_proto.h" +#include "utils/ms_utils.h" +#include "include/common/utils/utils.h" +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP //防止重复定义，如果当前宏是否被定义，若没有则定义，若已被定义，执行后面的语句 +#include "frontend/parallel/tensor_layout/tensor_layout.h" +#endif //ifndef条件命令结束标志 +#include "abstract/abstract_function.h" + +namespace mindspore { //创建一个名为mindspore的空间 +using FloatPtr = std::shared_ptr; //引入使用Float空间 +using IntPtr = std::shared_ptr; //引入使用Int空间 +using UIntPtr = std::shared_ptr; //引入使用UInt空间 +using ModelProtoPtr = std::shared_ptr; //引入使用ModelProto空间 + +// anf type to mindir type map 将数据类型转为mindir类型，以下同理 +static mindspore::HashMap g_data_type_map = { + {kNumberTypeBool, mind_ir::TensorProto_DataType_BOOL}, + {kNumberTypeInt8, mind_ir::TensorProto_DataType_INT8}, + {kNumberTypeInt16, mind_ir::TensorProto_DataType_INT16}, + {kNumberTypeInt32, mind_ir::TensorProto_DataType_INT32}, + {kNumberTypeInt64, mind_ir::TensorProto_DataType_INT64}, + {kNumberTypeUInt8, mind_ir::TensorProto_DataType_UINT8}, + {kNumberTypeUInt16, mind_ir::TensorProto_DataType_UINT16}, + {kNumberTypeUInt32, mind_ir::TensorProto_DataType_UINT32}, + {kNumberTypeUInt64, mind_ir::TensorProto_DataType_UINT64}, + {kNumberTypeFloat16, mind_ir::TensorProto_DataType_FLOAT16}, + {kNumberTypeFloat32, mind_ir::TensorProto_DataType_FLOAT}, + {kNumberTypeFloat64, mind_ir::TensorProto_DataType_DOUBLE}, + {kObjectTypeString, mind_ir::TensorProto_DataType_STRING}, + {kNumberTypeComplex64, mind_ir::TensorProto_DataType_COMPLEX64}, + {kNumberTypeComplex128, mind_ir::TensorProto_DataType_COMPLEX128}}; + +static mindspore::HashMap g_data_bits_int_map = { + {8, mind_ir::TensorProto_DataType_INT8}, + {16, mind_ir::TensorProto_DataType_INT16}, + {32, mind_ir::TensorProto_DataType_INT32}, + {64, mind_ir::TensorProto_DataType_INT64}, +}; + +static mindspore::HashMap g_data_bits_uint_map = { + {8, mind_ir::TensorProto_DataType_UINT8}, + {16, mind_ir::TensorProto_DataType_UINT16}, + {32, mind_ir::TensorProto_DataType_UINT32}, + {64, mind_ir::TensorProto_DataType_UINT64}, +}; + +static mindspore::HashMap g_data_bits_float_map = { + {16, mind_ir::TensorProto_DataType_FLOAT16}, + {32, mind_ir::TensorProto_DataType_FLOAT}, + {64, mind_ir::TensorProto_DataType_FLOAT64}, +}; + +static std::set g_export_attr_blacklist = {kAttrDump}; + +// Can build different builder according to format +class IrExportBuilder; //IrExport生成器类 +using IrExportBuilderPtr = std::shared_ptr; //引入使用IrExportBuilder空间 + +class IrExporter { + public: + explicit IrExporter(IrExportBuilderPtr builder) : builder_(std::move(builder)) {} //避免IrExporter隐式转换为类类型 + virtual ~IrExporter() = default; + std::string GetDumpString(const FuncGraphPtr &func_graph); + ModelProtoPtr GetDumpProto(const FuncGraphPtr &func_graph, const FuncGraphPtr ¶m_layout_fg = nullptr); + + private: + IrExportBuilderPtr builder_; +}; +using IrExporterPtr = std::shared_ptr; //引入使用IrExporterr空间 + +class IrExportBuilder { + public: + IrExportBuilder() : model_(std::make_shared()) {} + ~IrExportBuilder() = default; //默认构造函数 + std::string GetProtoString() const; + void BuildModelInfo(); + bool BuildModel(const FuncGraphPtr &func_graph); + ModelProtoPtr Model() { return model_; } + +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP //判断当前宏是否被定义 + void BuildLayout(const FuncGraphPtr &func_graph); +#endif +// + bool BuildFuncGraph(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); //图表 + bool BuildFuncGraphAttrs(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); //图表属性 + bool BuildParameters(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); //参数 + bool BuildNodes(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); //节点 + bool BuildOutput(const CNodePtr &node, mind_ir::GraphProto *const graph_proto); //输出 + bool BuildCNode(const CNodePtr &node, mind_ir::GraphProto *const graph_proto); //c节点 + std::string BuildInputNode(const AnfNodePtr &node, mind_ir::GraphProto *const graph_proto); //输入节点 +//判断是否set函数 + bool SetValueInfoProto(const AnfNodePtr &node, mind_ir::ValueInfoProto *const value_proto); + bool SetParamToTensorProto(const ParameterPtr ¶m, mind_ir::TensorProto *const tensor_proto); + bool SetTensorProto(const AbstractBasePtr &abstract, mind_ir::TensorProto *const tensor_proto); + bool SetCSRTensorToProto(const AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto); + bool SetAttributeProto(const AnfNodePtr &node, mind_ir::NodeProto *const node_proto); + bool SetAbstractToNodeProto(const CNodePtr &node, mind_ir::NodeProto *const node_proto); + bool SetAbstractToNodeProto(const abstract::AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto); + bool SetValueToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetTypeToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetScalarToAttributeProto_ir(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetScalarToAttributeProtoForInt_ir(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetScalarToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetScalarToAttributeProtoForInt_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetTypeToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetTensorToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetSequenceToAttributeProto(const ValueSequencePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetSeqElemToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); +//get函数 + mind_ir::TensorProto_DataType GetMindirDataType(TypeId type_id); + mind_ir::TensorProto_DataType GetMindirDataBitsIntType(int bits); + mind_ir::TensorProto_DataType GetMindirDataBitsFloatType(int bits); + mind_ir::TensorProto_DataType GetMindirDataBitsUIntType(int bits); + std::string GetNodeName(const AnfNodePtr &node); + std::string GetUniqueNodeName(const AnfNodePtr &node); + std::string GetOpTypeName(const AnfNodePtr &node); + size_t GetUniqueID() { return ++unique_id_; } + + private: + bool SetAbstractFuncToAttributeProto(const abstract::AbstractBasePtr &abstract, + mind_ir::AttributeProto *const attr_proto); + std::string GetPrimitiveUniqueName(const PrimitivePtr &primitive_ptr); + bool BuildPrimitives(); + + ModelProtoPtr model_; //创建名为model的ModelProto空间 + mind_ir::NodeProto *last_node_{nullptr}; + std::list todo_; //链表 + std::map node_name_map_; + std::map primitive_name_map_; + std::set nodeName_; + size_t unique_id_{0}; + bool top_graph{true}; +}; + +bool IrExportBuilder::SetAbstractFuncToAttributeProto(const abstract::AbstractBasePtr &abstract, + mind_ir::AttributeProto *const attr_proto) { + MS_EXCEPTION_IF_NULL(abstract); //捕捉是否是空指针异常 + MS_EXCEPTION_IF_NULL(attr_proto); + if (abstract->isa()) { //如果是函数图抽象闭包 + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_FUNCGRAPHCLOSURE);//设置类型 + auto func_name = abstract->cast()->func_graph()->ToString(); + attr_proto->set_s(func_name); + } else if (abstract->isa()) { //如果是原始抽象闭包 + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_PRIMITIVECLOSURE); //设置类型 + auto prim = abstract->cast()->prim(); + attr_proto->set_s(GetPrimitiveUniqueName(prim)); + } else if (abstract->isa()) { //如果是部分抽象闭包 + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_PARTIALCLOSURE); + auto node_ptr = abstract->cast()->node(); + MS_EXCEPTION_IF_NULL(node_ptr); //判断是否为空指针 + attr_proto->set_s(GetUniqueNodeName(node_ptr)); + } else if (abstract->isa()) { //如果是抽象函数并集 + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UNIONFUNCCLOSURE); + auto visit_func = [this, &attr_proto](const abstract::AbstractFuncAtomPtr &poss) { + auto element_attr_proto = attr_proto->add_values(); + if (!this->SetAbstractFuncToAttributeProto(poss, element_attr_proto)) { + MS_LOG(EXCEPTION) << "Set union function abstract to proto error." << poss->ToString(); //抛出异常 + } + }; + abstract->cast()->Visit(visit_func); + } else { + MS_LOG(ERROR) << "The parameter abstract is not an abstractFunction: " << abstract->ToString(); //抛出错误 + return false; + } + return true; +} + +std::string IrExportBuilder::GetPrimitiveUniqueName(const PrimitivePtr &primitive_ptr) { + auto it = primitive_name_map_.find(primitive_ptr); + if (it != primitive_name_map_.end()) { + return it->second; //it不等于primitive_name_map_.end()就指向下一个 + } + auto answer = primitive_ptr->name() + ":" + std::to_string(GetUniqueID()); //将name和ID存在answer里 + primitive_name_map_[primitive_ptr] = answer; //再存到原始数据指针里 + return answer; +} + +bool IrExportBuilder::BuildPrimitives() { //创建基本数据 + for (auto it = primitive_name_map_.begin(); it != primitive_name_map_.end(); ++it) { //遍历 + auto prim_proto = model_->add_primitives(); + auto prim = it->first; + prim_proto->set_name(it->second); + prim_proto->set_op_type(prim->name()); + + if (prim->isa()) { + auto func = prim->cast()->function(); + if (func != nullptr && func->isa()) { + prim = func->cast(); + } + } + + // Set primitive attributes + for (const auto &attr : prim->attrs()) { //遍历atrrs和基本属性 + MS_LOG(DEBUG) << "attr: " << attr.first << " " << attr.second->DumpText() << " " << attr.second->type_name(); + auto iter = g_export_attr_blacklist.find(attr.first); + if (iter != g_export_attr_blacklist.end()) { + continue; + } + mind_ir::AttributeProto *attr_proto = prim_proto->add_attribute(); + attr_proto->set_name(attr.first); + auto attr_value = attr.second; + CheckAndConvertUtils::ConvertAttrValueInExport(prim->name(), attr.first, &attr_value); + if (!SetValueToAttributeProto(attr_value, attr_proto)) { + MS_LOG(ERROR) << "Set value to AttributeProto failed."; //抛出错误 + return false; + } + } // Loop of attrs + } // Loop of primitives + return true; +} + +std::string IrExporter::GetDumpString(const FuncGraphPtr &func_graph) { + auto dump_proto = GetDumpProto(func_graph); + if (dump_proto == nullptr) { + MS_LOG(EXCEPTION) << "Get dump proto for graph " << func_graph->ToString() << " failed."; //空指针抛出异常 + } + return builder_->GetProtoString(); +} + +ModelProtoPtr IrExporter::GetDumpProto(const FuncGraphPtr &func_graph, const FuncGraphPtr ¶m_layout_fg) { + if ((builder_ == nullptr) || (func_graph == nullptr)) { + MS_LOG(EXCEPTION) << "Input params is null."; //空指针抛出异常 + } + + // Export model info 导出模型信息 + builder_->BuildModelInfo(); + + // Export model and return string 导出模型和返回字符串 + if (!builder_->BuildModel(func_graph)) { + return nullptr; + } + +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP + // Export layout information 导出布局信息 + if (param_layout_fg) { + builder_->BuildLayout(param_layout_fg); + } +#endif + return builder_->Model(); +} + +std::string IrExportBuilder::GetProtoString() const { + MS_LOG(DEBUG) << "BuildModel complete!"; //建立model完成 + return model_->SerializeAsString(); +} + +void IrExportBuilder::BuildModelInfo() { //利用set函数建立model中的基本信息 + constexpr auto ir_version = "0.1.1"; + constexpr auto mindspore_name = "MindSpore"; + model_->set_ir_version(ir_version); + model_->set_producer_name(mindspore_name); + model_->set_model_version(VERSION); + model_->set_little_endian(common::IsLittleByteOrder()); + model_->set_mind_ir_version(mind_ir::Version_MAX); +} + +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP //防止二次定义 +void IrExportBuilder::BuildLayout(const FuncGraphPtr &func_graph) { //设置布局 + MS_EXCEPTION_IF_NULL(func_graph); + std::vector graph_params = func_graph->parameters(); + mind_ir::ParallelProto *parallel_proto = model_->mutable_parallel(); + for (auto para : graph_params) { + std::string name = std::static_pointer_cast(para)->name(); + auto tensor_layout = para->user_data(); + if (tensor_layout == nullptr) { + MS_LOG(INFO) << "GetParameterLayout nullptr name = " << name; + } else { + mind_ir::LayoutProto *layoutProto = parallel_proto->add_layout(); //tensor_layout不为空，则添加布局 + + // Get all the information for layout 获取所有布局信息 + auto device_arrangement = tensor_layout->device_arrangement().array(); + auto tensor_map = tensor_layout->tensor_map().array(); + auto slice_shape = tensor_layout->slice_shape().array(); + int64_t field_size = tensor_layout->get_field_size(); + bool uniform_split = tensor_layout->uniform_split(); + std::string opt_shard_group = tensor_layout->opt_shard_group(); + + // Save all information to Layout Proto 将信息保存到Layou Proto中 + layoutProto->set_name(name); + for (auto device_arrangement_element : device_arrangement) { + layoutProto->add_device_arrangement_int(device_arrangement_element); + } + for (auto tensor_map_element : tensor_map) { + layoutProto->add_tensor_map_int(tensor_map_element); + } + for (auto slice_shape_element : slice_shape) { + layoutProto->add_slice_shape_int(slice_shape_element); + } + layoutProto->set_field_size(field_size); + layoutProto->set_uniform_split(uniform_split); + layoutProto->set_opt_shard_group(opt_shard_group); + } + } +} +#endif + +bool IrExportBuilder::BuildModel(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); //判断是否为空 + mind_ir::GraphProto *graph_proto = model_->mutable_graph(); + graph_proto->set_name(func_graph->ToString()); //用set函数设置信息 + graph_proto->set_bprop_hash(func_graph->bprop_hash()); + todo_.clear(); //清空 + nodeName_.clear(); + primitive_name_map_.clear(); + // Build the main funcGraph 建立主要funcGraph + (void)nodeName_.insert(func_graph->ToString()); + top_graph = true; + if (!BuildFuncGraph(func_graph, graph_proto)) { + MS_LOG(ERROR) << "Build func_graph " << func_graph->ToString() << " failed."; //抛出错误 + return false; + } + + // Build child funcGraphs 建立子funcGraph + std::set graphVisited; + (void)graphVisited.insert(func_graph); + top_graph = false; + while (!todo_.empty()) { + FuncGraphPtr fg = todo_.back(); + todo_.pop_back(); + if (graphVisited.count(fg) > 0) { + continue; //若满足条件，则跳过该语句执行下一条 + } + if (nodeName_.count(fg->ToString()) > 0) { + MS_LOG(ERROR) << "There is a duplicate name: " << fg->ToString(); //当名称重复时抛出错误 + return false; + } + (void)nodeName_.insert(fg->ToString()); + (void)graphVisited.insert(fg); + auto graph = model_->add_functions(); + if (!BuildFuncGraph(fg, graph)) { + MS_LOG(ERROR) << "Build func_graph " << fg->ToString() << " failed."; //抛出错误 + return false; + } + } + + if (!BuildPrimitives()) { + return false; + } + // Release resource 释放空间 + nodeName_.clear(); + node_name_map_.clear(); + primitive_name_map_.clear(); + return true; +} +//构造函数图 +bool IrExportBuilder::BuildFuncGraph(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + // Export funcGraph name.到处funcGraph的名字 + graph_proto->set_name(func_graph->ToString()); + // Export parameters 导出参数 + // 1. parameters should be mapped to ValueInfoProto 参数应映射到ValueInfoProto + // 2. parameters with default value should be mapped to Initializer 省略残缺的参数应映射到初始值设定项 + if (!BuildParameters(func_graph, graph_proto)) { + MS_LOG(ERROR) << "Build parameters failed."; //抛出错误创建参数失败 + return false; + } + + // Export graph attributes 导出图形属性 + if (!BuildFuncGraphAttrs(func_graph, graph_proto)) { + MS_LOG(ERROR) << "Build attributes for graph failed."; + return false; + } + + // Export operator nodes(include output) 导出运算符节点（包括输出） + return BuildNodes(func_graph, graph_proto); +} +//构造函数图属性 +bool IrExportBuilder::BuildFuncGraphAttrs(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + MS_EXCEPTION_IF_NULL(func_graph); //判断是否为空 + MS_EXCEPTION_IF_NULL(graph_proto); + for (auto attr : func_graph->attrs()) { + MS_LOG(DEBUG) << "attr: " << attr.first << " " << attr.second->DumpText() << " " << attr.second->type_name(); + auto iter = g_export_attr_blacklist.find(attr.first); + if (iter != g_export_attr_blacklist.end()) { //遍历 + continue; + } + mind_ir::AttributeProto *attr_proto = graph_proto->add_attribute(); + attr_proto->set_name(attr.first); //设置名字 + if (!SetValueToAttributeProto(attr.second, attr_proto)) { + MS_LOG(ERROR) << "Set value to AttributeProto for GraphProto failed."; //将GraphProto的值设置为AttributeProto失败 + return false; + } + } + return true; +} +//构建参数 +bool IrExportBuilder::BuildParameters(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(graph_proto); + for (auto &item : func_graph->parameters()) { + MS_EXCEPTION_IF_NULL(item); + auto param = item->cast(); //将cast()赋给param + if (param == nullptr) { + MS_LOG(ERROR) << "Parameter: '" << item->ToString() << "' could not cast to parameter."; //如果为空就报错 + return false; + } + std::string param_name = GetUniqueNodeName(param); //用get函数提出param的name + if (top_graph && param->has_default()) { + MS_LOG(DEBUG) << "Parameter: '" << item->DebugString(); + mind_ir::TensorProto *parameter_proto = graph_proto->add_parameter(); //用add函数将parameter存入张量中 + parameter_proto->set_name(param_name); + if (!SetParamToTensorProto(param, parameter_proto)) { + MS_LOG(ERROR) << "Set parameter " << param->DebugString() << " to TensorProto failed."; //报错，将parameter存入张量失败 + return false; + } + } else { + mind_ir::ValueInfoProto *input_proto = graph_proto->add_input(); + input_proto->set_name(param_name); //设置名称 + if (!SetValueInfoProto(param, input_proto)) { + MS_LOG(ERROR) << "Set parameter " << param->DebugString() << " to TensorProto failed."; //报错 + return false; + } + } + if (nodeName_.count(param_name) > 0) { + MS_LOG(ERROR) << "parameter name is duplicate:" << param_name; //参数名称重复 + return false; + } + (void)nodeName_.insert(param_name); //插入 + } + return true; +} + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataType(TypeId type_id) { //获取Mindir数据类型 + auto iter = g_data_type_map.find(type_id); + if (iter == g_data_type_map.end()) { + MS_LOG(ERROR) << "Convert type error, unsupported type! " << type_id; //如果iter是g_data_type_map的结尾，则转换类型错误，该类型不支持 + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataBitsIntType(int bits) { //获取Int类型的Mindir数据 + auto iter = g_data_bits_int_map.find(bits); + if (iter == g_data_bits_int_map.end()) { + MS_LOG(ERROR) << "Convert bits int error, unsupported bits! " << bits; //转换错误 + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataBitsUIntType(int bits) { //获取UInt类型的Mindir数据位 + auto iter = g_data_bits_uint_map.find(bits); + if (iter == g_data_bits_uint_map.end()) { + MS_LOG(ERROR) << "Convert bits uint error, unsupported bits! " << bits; //转换错误 + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataBitsFloatType(int bits) { //获取Float类型的Mindir数据位 + auto iter = g_data_bits_float_map.find(bits); + if (iter == g_data_bits_float_map.end()) { + MS_LOG(ERROR) << "Convert bits float error, unsupported bits! " << bits; //转换错误 + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} +//判断是否设置ValueInfoProto +bool IrExportBuilder::SetValueInfoProto(const AnfNodePtr &node, mind_ir::ValueInfoProto *const value_proto) { + if (node == nullptr || value_proto == nullptr) { + MS_LOG(EXCEPTION) << "AnfNode or ValueInfo is null!"; //空指针异常 + } + MS_LOG(DEBUG) << "SetValueInfoProto: " << node->DebugString(); //调试 + const TypePtr &type = node->Type(); + const BaseShapePtr &shape = node->Shape(); + // For the bprop fg which has not been renormalized. + if (type == nullptr || shape == nullptr) { + return true; + } + if (type->isa() && shape->isa()) { + mind_ir::TensorProto *tensor_proto = value_proto->add_tensor(); + if (!SetTensorProto(node->abstract(), tensor_proto)) { + return false; //如果未设置TensorProto，则返回false + } + } else if (type->isa()) { + mind_ir::AttributeProto *attribute = value_proto->mutable_attr_info(); + if (!SetAbstractToNodeProto(node->abstract(), attribute)) { + MS_LOG(ERROR) << "Set shape to Proto for " << node->DebugString() << " failed."; //设置形状报错 + return false; + } + attribute->set_name("shape"); //设置名称 + } else { + value_proto->set_denotation(type->type_name()); + } + MS_LOG(DEBUG) << "Value type: " << type->type_name(); //调试 + return true; +} +//判断是否将张量设置为属性Proto +bool IrExportBuilder::SetTensorToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; //空指针异常 + } + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); //设置类型 + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); //添加张量 + tensor_proto->set_name("value0"); //设置名称 + auto data = value->cast(); + MS_EXCEPTION_IF_NULL(data); //判断是否为空 + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); //设置原始数据 + auto dtype = data->data_type(); //类型 + auto shape = data->shape_c(); //形状 + auto data_type = GetMindirDataType(dtype); //数据类型 + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; //如果数据类型和未定义数据类型相同，则返回false + } + tensor_proto->set_data_type(data_type); + for (const auto &dim : shape) { + tensor_proto->add_dims(dim); + } + return true; +} +//判断是否将CSR张量设置为Proto +bool IrExportBuilder::SetCSRTensorToProto(const AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto) { + abstract::AbstractCSRTensorPtr csr_tensor_abs = abstract->cast(); + MS_EXCEPTION_IF_NULL(csr_tensor_abs); //判断是否为空 + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_CSR_TENSOR); //设置类型 + (void)SetTensorProto(csr_tensor_abs->indptr(), attr_proto->add_tensors()); //设置张量原型 + (void)SetTensorProto(csr_tensor_abs->indices(), attr_proto->add_tensors()); + (void)SetTensorProto(csr_tensor_abs->values(), attr_proto->add_tensors()); + auto dense_proto = attr_proto->add_values(); + return SetAbstractToNodeProto(csr_tensor_abs->dense_shape(), dense_proto); +} +//判断是否设置张量proto +bool IrExportBuilder::SetTensorProto(const AbstractBasePtr &abstract, mind_ir::TensorProto *const tensor_proto) { + auto type = abstract->BuildType(); //创建类型 + auto shape = abstract->BuildShape(); //创建形状 + if (!type->isa() || !shape->isa()) { + MS_LOG(ERROR) << "Type or shape is not supported! " << type->ToString(); //报错，类型或形状错误 + return false; + } + auto tensor = type->cast(); //张量类型指针 + auto tensor_shape = shape->cast(); //形状指针 + const auto &dims = tensor_shape->shape(); //形状常量 + auto data_type = GetMindirDataType(tensor->element()->type_id()); //获取Mindir数据类型 + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; //如果数据类型和未定义数据类型相同，则返回false + } + tensor_proto->set_data_type(data_type); + for (const auto &dim : dims) { + tensor_proto->add_dims(dim); + } + if (tensor_shape->IsDynamic()) { + auto min_shape = tensor_shape->min_shape(); //最小形状 + auto max_shape = tensor_shape->max_shape(); //最大形状 + for (auto item : min_shape) { + tensor_proto->add_min_dims(item); + } + for (auto item : max_shape) { + tensor_proto->add_max_dims(item); + } + } + // Deal Ref + if (!type->isa()) { + return true; + } + + auto abs_ref = abstract->cast(); + if (abs_ref == nullptr) { + MS_LOG(ERROR) << "The abstract " << abstract->ToString() << " should be AbstractRef."; //抽象引用 + return false; + } + auto ref_key_value = abs_ref->ref_key_value(); + if (ref_key_value == nullptr) { + MS_LOG(INFO) << "The ref_key_value of abstract ref " << abstract->ToString() << " is nullptr"; //空指针 + return true; + } + tensor_proto->set_ref_key(ref_key_value->name()); + return true; +} + +bool IrExportBuilder::SetParamToTensorProto(const ParameterPtr ¶m, mind_ir::TensorProto *const tensor_proto) { + if (param == nullptr || tensor_proto == nullptr) { + MS_LOG(EXCEPTION) << "Parameter or TensorProto is null!"; //空指针异常 + } + MS_LOG(DEBUG) << "SetParamToTensorProto: " << param->DebugString(); //调试 + return SetTensorProto(param->abstract(), tensor_proto); +} +//判断是否建立节点 +bool IrExportBuilder::BuildNodes(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + for (const AnfNodePtr &node : nodes) { + MS_EXCEPTION_IF_NULL(node); //排除空指针 + if (!node->isa()) { + MS_LOG(DEBUG) << "Node: '" << node->ToString() << "' is not cnode"; + continue;//不是cnode，执行下面语句 + } + auto cnode = node->cast(); + if (cnode == func_graph->get_return()) { + if (!BuildOutput(cnode, graph_proto)) { + MS_LOG(ERROR) << "Build output for graph " << func_graph->ToString() << " failed."; //图形的生成输出失败 + return false; + } + } else { + if (!BuildCNode(cnode, graph_proto)) { + MS_LOG(ERROR) << "Build proto for cnode " << cnode->DebugString() << " failed."; //建立节点失败 + return false; + } + } + } + return true; +} +//判断是否建立输出 +bool IrExportBuilder::BuildOutput(const CNodePtr &node, mind_ir::GraphProto *const graph_proto) { + MS_EXCEPTION_IF_NULL(node); //排除空指针 + const int OutputSize = 2; + if (node->size() != OutputSize) { + MS_LOG(ERROR) << "Number of inputs of return node is not equal to 2."; //返回节点的输入数不等于2 + return false; + } + AnfNodePtr arg = node->input(1); + std::string node_name = BuildInputNode(arg, graph_proto); + if (node_name.empty()) { + MS_LOG(ERROR) << "Build input node failed for arg " << arg->DebugString(); //若为空则报错：参数生成输入节点失败 + return false; + } + mind_ir::ValueInfoProto *output_proto = graph_proto->add_output(); + output_proto->set_name(node_name); + return SetValueInfoProto(arg, output_proto); +} + +std::string IrExportBuilder::GetOpTypeName(const AnfNodePtr &node) { //获取OpTypeName + // May be ValueNode/CNode/Parameter + std::string type_name = ""; + if (IsValueNode(node)) { + PrimitivePtr prim = GetValueNode(node); + MS_EXCEPTION_IF_NULL(prim); //排除空指针 + type_name = "REF::" + GetPrimitiveUniqueName(prim); + } else if (IsValueNode(node)) { + FuncGraphPtr fg = GetValueNode(node); + MS_EXCEPTION_IF_NULL(fg); //排除空指针 + todo_.push_back(fg); + type_name = "REF::" + fg->ToString(); + } else if (node->isa() || node->isa()) { + auto nodeName = GetUniqueNodeName(node); + type_name = "REF::" + nodeName; + if (nodeName_.count(nodeName) == 0) { + MS_LOG(ERROR) << "There is not the name: " << nodeName; //若为0.则没有名字 + return ""; + } + } else { + MS_LOG(ERROR) << "Need to support op type: " << node->type_name(); + return ""; + } + MS_LOG(DEBUG) << "ExportType: " << type_name; + return type_name; +} +//判断Node Proto是否设置抽象 +bool IrExportBuilder::SetAbstractToNodeProto(const AbstractBasePtr &abs, mind_ir::AttributeProto *const attr_proto) { + auto type = abs->BuildType(); + auto shape = abs->BuildShape(); + if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TUPLE); //设置类型 + auto tuple_abs = abs->cast(); + for (size_t i = 0; i < tuple_abs->size(); i++) { + mind_ir::AttributeProto *attr_values = attr_proto->add_values(); + if (!SetAbstractToNodeProto((*tuple_abs)[i], attr_values)) { + return false; + } + } + } else if (type->isa() && shape->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + return SetTensorProto(abs, tensor_proto); + } else if (type->isa()) { + if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_BOOL); + } else { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto data_type = GetMindirDataType(type->type_id()); + tensor_proto->set_data_type(data_type); + tensor_proto->add_dims(1); + } + } else if (type->isa()) { + if (!SetAbstractFuncToAttributeProto(abs, attr_proto)) { + return false; + } + } else if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_STRING); + } else if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UMONAD); + } else if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_IOMONAD); + } else if (type->isa()) { + auto csr_tensor_abs = abs->cast(); + if (!SetCSRTensorToProto(csr_tensor_abs, attr_proto)) { + return false; + } + } else { + MS_LOG(ERROR) << "Type of cnode need to be supported: " << type->type_name(); + return false; + } + return true; +} + +bool IrExportBuilder::SetAbstractToNodeProto(const CNodePtr &node, mind_ir::NodeProto *const node_proto) { + // Get shape of cnode 获得cnode的形状 + // 1. need to get shape from tuple element 需要从元组元素获取形状 + // 2. save shape in TensorProto 在TensorProto中保存形状 + MS_EXCEPTION_IF_NULL(node); + auto type = node->Type(); + auto shape = node->Shape(); + auto abs = node->abstract(); + // For the bprop fg which has not been renormalized. + if (type == nullptr || shape == nullptr) { + return true; + } + mind_ir::AttributeProto *attr_proto = node_proto->add_attribute(); + if (!SetAbstractToNodeProto(abs, attr_proto)) { + MS_LOG(ERROR) << "Set shape to NodeProto for " << node->DebugString() << " failed."; + return false; + } + attr_proto->set_name("shape"); + return true; +} +//判断是否建立CNode +bool IrExportBuilder::BuildCNode(const CNodePtr &node, mind_ir::GraphProto *const graph_proto) { + auto inputs_size = node->size(); + if (inputs_size < 1) { + MS_LOG(ERROR) << "Inputs of node " << node->DebugString() << " is empty"; //报错：节点输入为空 + return false; + } + + // Need to build input node before dealing with cnode 在处理cnode之前需要构建输入节点 + std::vector input_names; + for (size_t i = 1; i < inputs_size; i++) { + auto input = node->input(i); + std::string node_name = BuildInputNode(input, graph_proto); + if (node_name.empty()) { + MS_LOG(ERROR) << "Build input node for " << input->DebugString() << " failed."; //报错构建输入节点失败 + return false; + } + input_names.push_back(node_name); + } + + // Build cnode 创建cnode + mind_ir::NodeProto *node_proto = graph_proto->add_node(); + std::string output_name = GetUniqueNodeName(node); + if (nodeName_.count(output_name) > 0) { + MS_LOG(EXCEPTION) << "There is a duplicate name: " << output_name; + } + (void)nodeName_.insert(output_name); + node_proto->add_output(output_name); + node_proto->set_name(output_name); + node_proto->set_domain(node->fullname_with_scope()); + AnfNodePtr op = node->input(0); + std::string type_name = GetOpTypeName(op); + if (type_name.empty()) { + MS_LOG(ERROR) << "Get op type name for " << op->DebugString() << " failed."; + return false; + } + node_proto->set_op_type(type_name); + last_node_ = node_proto; + // Maybe Tensor or Function or nullptr 可能是张量、函数或空指针 + if (!SetAbstractToNodeProto(node, node_proto)) { + return false; + } + + (void)std::for_each(input_names.begin(), input_names.end(), + [&node_proto](const string &name) { node_proto->add_input(name); }); + return true; +} +//构建输入节点 +std::string IrExportBuilder::BuildInputNode(const AnfNodePtr &node, mind_ir::GraphProto *const graph_proto) { + // Return the NodeName that the node has been processed.返回节点已处理的节点名。 + auto iter = node_name_map_.find(node); + if (iter != node_name_map_.end()) { + return iter->second; + } + + std::string node_name = GetUniqueNodeName(node); + // FuncGraph will be added to functions and the input name is the function name. FuncGraph将添加到函数中，输入名称为函数名。 + if (IsValueNode(node)) { + FuncGraphPtr fg = GetValueNode(node); + todo_.push_back(fg); + return fg->ToString(); + } + if (node->isa()) { + (void)nodeName_.insert(node_name); + // When node input is a ValueNode, need to create a Constant Node 当节点输入是ValueNode时，需要创建一个常量节点 + mind_ir::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_name(node_name); + node_proto->add_output(node_name); + if (!SetAttributeProto(node, node_proto)) { + return ""; + } + } + return node_name; +} +//获取特殊节点名称 +std::string IrExportBuilder::GetUniqueNodeName(const AnfNodePtr &node) { + // Naming anfnode 命名节点 + // 1. parameter is unique in one func_graph 参数在一个func_ + // 2. cnode and valuenode may be reduplicative, so add index to identify. cnode和valuenode可能是重复的，因此添加索引以识别 + auto iter = node_name_map_.find(node); + if (iter != node_name_map_.end()) { + return iter->second; + } else { + std::string node_name = GetNodeName(node); + // Compatible before. CNode = FuncGraphName:CNodeName:index ,Parameter = FuncGraphName:ParameterName + if (node->isa()) { + node_name = node_name + ":" + std::to_string(GetUniqueID()); + } + // Avoid duplicate name. 避免名称重复 + while (nodeName_.count(node_name) > 0) { + node_name = node_name + "_" + std::to_string(GetUniqueID()); + } + node_name_map_[node] = node_name; + return node_name; + } +} +//获取节点名称 +std::string IrExportBuilder::GetNodeName(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + std::string node_name = ""; + if (node->func_graph() != nullptr) { + node_name = node->func_graph()->ToString() + ":"; + } + if (node->isa()) { + // Needn't value + node_name += node->AnfNode::ToString(); + } else { + node_name += node->ToString(); + } + MS_LOG(DEBUG) << "GetNodeName: " << node_name; + return node_name; +} + +bool IrExportBuilder::SetAttributeProto(const AnfNodePtr &node, mind_ir::NodeProto *const node_proto) { + if (node == nullptr || node_proto == nullptr) { + MS_LOG(EXCEPTION) << "AnfNode or NodeProto is null!"; //若为空指针则抛出异常 + } + auto value_node = node->cast(); + MS_EXCEPTION_IF_NULL(value_node);//排除空指针 + auto value = value_node->value(); + node_proto->set_op_type("Constant"); //设置op类型 + mind_ir::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); //设置名称 + MS_LOG(DEBUG) << "Set Constant attribute: " << value->ToString(); //调试 + return SetValueToAttributeProto(value, attr_proto); +} + +bool IrExportBuilder::SetTypeToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; //若为空指针则抛出异常 + } + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); //设置类型 + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); //添加张量 + //判断int型、uint型、float型、bool型、tensor型 + if (value->isa()) { + tensor_proto->set_name("value0"); //设置名称 + auto int_value = value->cast(); + auto data_type = GetMindirDataBitsIntType(int_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { //如果数据类型未定义，则返回false + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + tensor_proto->set_name("value0"); + auto float_value = value->cast(); + auto data_type = GetMindirDataBitsUIntType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + tensor_proto->set_name("value0"); + auto float_value = value->cast(); + auto data_type = GetMindirDataBitsFloatType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + tensor_proto->set_name("value0"); + tensor_proto->set_data_type(mind_ir::TensorProto_DataType_BOOL); + } else if (value->isa()) { + tensor_proto->set_name("tensor0"); + auto elem_type = value->cast()->element(); + if (elem_type->isa()) { + auto int_value = elem_type->cast(); + auto data_type = GetMindirDataBitsIntType(int_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (elem_type->isa()) { + auto float_value = elem_type->cast(); + auto data_type = GetMindirDataBitsFloatType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else { + MS_LOG(ERROR) << "Unsupported type " << elem_type->type_name(); + return false; + } + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << value->type_name(); //若都不是，则异常：没有支持的类型 + return false; + } + return true; +} + +bool IrExportBuilder::SetValueToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; //空指针 + } + //可能是stringimm或scalar、number或tensortype、ValueSequence、Tensor、None、Monad、IOMonad + if (value->isa() || value->isa()) { + return SetScalarToAttributeProto_ir(value, attr_proto); + } else if (value->isa() || value->isa()) { + return SetTypeToAttributeProto(value, attr_proto); + } else if (value->isa()) { + if (!SetSequenceToAttributeProto(value->cast(), attr_proto)) { + MS_LOG(ERROR) << "Set sequence to AttributeProto failed."; //报错：将序列设置为AttributeProto失败 + return false; + } + MS_LOG(DEBUG) << "Attr string: " << value->type_name(); //调试 + } else if (value->isa()) { + return SetTensorToAttributeProto(value, attr_proto); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_NONE); + MS_LOG(DEBUG) << "Attr string: " << value->type_name(); + } else if (value->isa()) { + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UMONAD); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_IOMONAD); + } else { + MS_LOG(ERROR) << "Unsupported Monad type: " << value->type_name(); + return false; //不支持的Monad类型，返回false + } + } else { + MS_LOG(ERROR) << "Unsupported type: " << value->type_name(); //报错：没有支持的类型，返回false + return false; + } + return true; +} + +bool IrExportBuilder::SetScalarToAttributeProto_ir(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; //空指针 + } + //可能是StringImm、BoolImm、Int、FP32Imm、FP64Imm + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_STRING); + attr_proto->set_s(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_BOOL); + int64_t attr_value = GetValue(value) ? 1 : 0; + attr_proto->set_i(attr_value); + } else if (SetScalarToAttributeProtoForInt_ir(value, attr_proto)) { + return true; + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_FLOAT); + attr_proto->set_f(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_DOUBLE); + attr_proto->set_d(GetValue(value)); + } else { + MS_LOG(ERROR) << "Unsupported scalar type: " << value->type_name(); + return false; //不支持的scalar类型，返回false + } + return true; +} + +bool IrExportBuilder::SetScalarToAttributeProtoForInt_ir(const ValuePtr &value, + mind_ir::AttributeProto *const attr_proto) { + //可能是Int8Imm、Int16Imm、Int32Imm、Int64Imm、UInt8Imm、UInt16Imm、UInt32Imm、UInt64Imm + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT8); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT16); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT32); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT64); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT8); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT16); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT32); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT64); + attr_proto->set_i(UlongToLong(value->cast()->value())); + } else { + return false; + } + return true; +} + +bool IrExportBuilder::SetTypeToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "AttributeProto is null!";//空指针 + } + //可能是Int、Float、UInt、Bool、Tensor + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto int_value = value->cast(); + auto data_type = GetMindirDataBitsIntType(int_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto float_value = value->cast(); + auto data_type = GetMindirDataBitsFloatType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto uint_value = value->cast(); + auto data_type = GetMindirDataBitsFloatType(uint_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + tensor_proto->set_data_type(mind_ir::TensorProto_DataType_BOOL); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + return SetTensorToAttributeProto(value, attr_proto); + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << value->type_name(); + return false; //不支持的类型，返回false + } + return true; +} + +bool IrExportBuilder::SetScalarToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "AttributeProto is null!"; //空指针 + } + 可能是StringImm、BoolImm、FP32Imm、FP64Imm + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_STRING); + attr_proto->add_strings(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_BOOL); + attr_proto->add_ints(GetValue(value)); + } else if (SetScalarToAttributeProtoForInt_irs(value, attr_proto)) { + return true; + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_FLOAT); + attr_proto->add_floats(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_DOUBLE); + attr_proto->add_doubles(GetValue(value)); + } else { + MS_LOG(ERROR) << "Unsupported scalar type: " << value->type_name(); + return false; //不支持的scalar类型，返回false + } + return true; +} + +bool IrExportBuilder::SetScalarToAttributeProtoForInt_irs(const ValuePtr &value, + mind_ir::AttributeProto *const attr_proto) { + //可能是Int8Imm、Int16Imm、Int32Imm、Int64Imm、UInt8Imm、UInt16Imm、UInt32Imm、UInt64Imm + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT8); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT16); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT32); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT64); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT8); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT16); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT32); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT64); + attr_proto->add_ints(SizeToInt(value->cast()->value())); + } else { + return false; + } + return true; +} +//判断是否将将Seq Elem设置为AttributeProto +bool IrExportBuilder::SetSeqElemToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr) { + MS_LOG(ERROR) << "Value is nullptr";//空指针 + return false; + } + if (value->isa() || value->isa()) { + return SetScalarToAttributeProto_irs(value, attr_proto); //若为StringImm或Scalar，则返回SetScalarToAttributeProto_irs(value, attr_proto) + } + return SetTypeToAttributeProto_irs(value, attr_proto); +} +//判断是否将序列设置为AttributeProto +bool IrExportBuilder::SetSequenceToAttributeProto(const ValueSequencePtr &value, + mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValueSequencePtr or AttributeProto is null!"; //空 + } + //可能是ValueTuple、ValueList + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TUPLE); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_LIST); + } else { + MS_LOG(EXCEPTION) << "The sequance value should be ValueTuple or ValueList, but it is " << value->ToString(); //sequance值应该是ValueTuple或ValueList + } + auto value_sequence = value->cast(); + MS_EXCEPTION_IF_NULL(value_sequence); //排除空指针 + const auto &values = value_sequence->value(); + if (values.empty()) { + MS_LOG(DEBUG) << "SetSequenceToAttributeProto sequence size is 0"; //如果为空，则调试SetSequenceToAttributeProto序列大小为0 + return true; + } + for (const auto &item : values) { + mind_ir::AttributeProto *attr_values = attr_proto->add_values(); + MS_EXCEPTION_IF_NULL(item); //排除空指针 + if (item->isa()) { + if (!SetSequenceToAttributeProto(item->cast(), attr_values)) { + MS_LOG(ERROR) << "Set sequence to AttributeProto failed."; + return false; + } + } else { + if (!SetSeqElemToAttributeProto(item, attr_values)) { + MS_LOG(ERROR) << "Set seq elem to AttributeProto failed."; + return false; + } + } + } + return true; +} +//获取二进制原型字符串 +std::string GetBinaryProtoString(const FuncGraphPtr &func_graph) { + auto builder = std::make_shared(); + if (builder == nullptr) { + MS_LOG(ERROR) << "Create ir exporter failed!"; //若为空则创建ir导出器失败 + return ""; + } + auto exporter = std::make_shared(builder); + if (exporter == nullptr) { + return ""; + } + auto ret = exporter->GetDumpString(func_graph); + return ret; +} +//是否转储二进制proto +bool DumpBinaryProto(const FuncGraphPtr &func_graph, const std::string &file_path, + const FuncGraphPtr ¶m_layout_fg) { + auto exporter = std::make_shared(std::make_shared()); + auto proto = exporter->GetDumpProto(func_graph, param_layout_fg); + if (proto == nullptr) { + MS_LOG(ERROR) << "Get binary proto for graph " << func_graph->ToString() << " failed."; + return false; + } + + auto realpath = Common::CreatePrefixPath(file_path, true); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path of file " << file_path << " failed."; + return false; + } + + ChangeFileMode(realpath.value(), S_IWUSR); + std::ofstream fout(realpath.value()); + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open the file '" << realpath.value() << "' failed!" << ErrnoToString(errno); + return false; + } + + if (!proto->SerializeToOstream(&fout)) { + MS_LOG(ERROR) << "Failed to write the mindir proto to file " << realpath.value(); + fout.close(); + return false; + } + fout.close(); + ChangeFileMode(realpath.value(), S_IRUSR); + return true; +} +} // namespace mindspore -- 2.34.1 From 5d2062f6bdf2d13b29347ad8bd18c963e82afd3f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:36:32 +0800 Subject: [PATCH 071/160] ADD file via upload --- .../ccsrc/transform/graph_ir/util.cc | 549 ++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/util.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/util.cc b/mindspore2022/ccsrc/transform/graph_ir/util.cc new file mode 100644 index 00000000000..eda89bd3759 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/util.cc @@ -0,0 +1,549 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/util.h" + +#include +#include + +#include "securec/include/securec.h" +#include "include/common/utils/convert_utils.h" +#include "include/common/utils/utils.h" + +namespace mindspore { +namespace transform { +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; + +const size_t kErrorSize = 0; +//在TransformUtil区域中创建名为ConvertIntToList的函数，返回值类型为vector，用于检测size的长度。 +//若长度小于等于零则弹出MS_LOG警告日志 "size <= 0"并直接返回list +//若长度大于零则用list对data中的内容进行push_back操作 +vector TransformUtil::ConvertIntToList(int64_t data, int size) { + vector list{}; + if (size <= 0) { + MS_LOG(WARNING) << "size <= 0"; + return list; + } + for (int i = 0; i < size; ++i) { + list.push_back(data); + } + return list; +} + +static std::map datatype_trans_map = { + {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, + {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, + {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, + {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, + {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, + {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; +//在TransformUtil区域中创建名为ConvertDataType的函数，返回值类型为自定义GeDataType +//输出MS_LOG调试日志"Convert me data type: "[*此处为对type进行TypeIdLabel(type)的结果]ge data type" +//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 +//若不一致，则返回datatype_trans_map[type] +//若一致则返回DT_UNDEFINED +GeDataType TransformUtil::ConvertDataType(const MeDataType &type) { + MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; + if (datatype_trans_map.find(type) != datatype_trans_map.end()) { + return datatype_trans_map[type]; + } else { + return GeDataType::DT_UNDEFINED; + } +} + +static std::map datatype_size_map = { + {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float + {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, + {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, + {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, + {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, + {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; +//在TransformUtil区域中创建名为GetDataTypeSize的函数，返回值类型为自定义size_t +//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 +//若不一致，则返回datatype_trans_map[type] +//若一致，则输出错误日志 "Illegal tensor data type!"并返回kErrorSize +size_t TransformUtil::GetDataTypeSize(const MeDataType &type) { + if (datatype_size_map.find(type) != datatype_size_map.end()) { + return datatype_size_map[type]; + } else { + MS_LOG(ERROR) << "Illegal tensor data type!"; + return kErrorSize; + } +} +//在TransformUtil区域中创建名为:ConvertFormat的函数，返回值类型为自定义GeFormat +//判断format与 kOpFormat_NCHW等kOpFormat_*是否相等，若相等，则返回 kOpFormat_NCHW等kOpFormat_* +//若与一系列kOpFormat_*都不相等，则输出错误日志 "Illegal tensor data format: ("[*此处为format变量]"). Use ND format instead." +//并返回FORMAT_ND +GeFormat TransformUtil::ConvertFormat(const string &format) { + if (format == kOpFormat_NCHW) { + return GeFormat::FORMAT_NCHW; + } else if (format == kOpFormat_NDHWC) { + return GeFormat::FORMAT_NDHWC; + } else if (format == kOpFormat_NCDHW) { + return GeFormat::FORMAT_NCDHW; + } else if (format == kOpFormat_DHWNC) { + return GeFormat::FORMAT_DHWNC; + } else if (format == kOpFormat_DHWCN) { + return GeFormat::FORMAT_DHWCN; + } else if (format == kOpFormat_NC1HWC0) { + return GeFormat::FORMAT_NC1HWC0; + } else if (format == kOpFormat_NHWC) { + return GeFormat::FORMAT_NHWC; + } else if (format == kOpFormat_HWCN) { + return GeFormat::FORMAT_HWCN; + } else if (format == kOpFormat_ND) { + return GeFormat::FORMAT_ND; + } else { + MS_LOG(ERROR) << "Illegal tensor data format: (" << format << "). Use ND format instead."; + return GeFormat::FORMAT_ND; + } +} +//创建名为IntegerCastFunc的函数，返回值类型为自定义static int64_t +//返回执行static_cast()操作后的size_t的temp +static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } + +//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr TransformUtil::GetGeTensorDesc(const ShapeVector &me_shape, const MeDataType &me_type, + const std::string &format) { + // convert me shape to ge shape + //将me_shape转化为ge_shape + //定义ge_shape + std::vector ge_shape; + //若me_shape.size()=1，则对其第[0]位进行push_back操作 + //否则，则根据me_shape的size属性设置ge_shape的size属性 + if (me_shape.size() == 1) { + ge_shape.push_back(static_cast(me_shape[0])); + } else { + ge_shape.resize(me_shape.size()); + (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); + } + //若shape.GetDimNum()=0，则输出信息日志"The dims size of Ge tensor is zero" + GeShape shape(ge_shape); + if (shape.GetDimNum() == 0) { + MS_LOG(INFO) << "The dims size of Ge tensor is zero"; + } + // convert me format to ge format + //将me_format转化为ge_format + //若ge_format符合相应条件则输出信息日志 "Set ND data format" + GeFormat ge_format = ConvertFormat(format); + if (ge_format == GeFormat::FORMAT_ND) { + MS_LOG(INFO) << "Set ND data format"; + } + // convert me datatype to ge datatype + //将me_datatype转化为ge_datatype + //若data_type符合相应条件则输出错误日志"undefined data type :"[*此处为me_type] + //并返回 nullptr + GeDataType data_type = ConvertDataType(me_type); + if (data_type == GeDataType::DT_UNDEFINED) { + MS_LOG(ERROR) << "undefined data type :" << me_type; + return nullptr; + } + //进行make_shared操作，若desc变量为空则输出错误日志 "Create GeTensorDesc failed!" + //并返回 nullptr + //若不为空则输出信息日志"SetRealDimCnt is :"[*此处为 me_shape.size()] + //并对desc进行SetRealDimCnt操作 + //并返回desc + //这一步在检验转换是否成功并且转换后对象的size属性正确 + auto desc = std::make_shared(shape, ge_format, data_type); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create GeTensorDesc failed!"; + return nullptr; + } + MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); + desc->SetRealDimCnt(SizeToInt(me_shape.size())); + return desc; +} + +// if failed, return empty vector. +//如果失败了，则返回空的vector +//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr +std::vector TransformUtil::ConvertInputTensors(const std::vector &me_tensors, + const std::string &format) { + //定义ge_tensors + std::vector ge_tensors; + //在index小于me_tensors.size的范围内递增循环 + //输出消息日志 "me_tensor "[*此处为index(tensors的位数)]" 's data size is: "[*此处为index对应位的tensors的size] + for (size_t index = 0; index < me_tensors.size(); index++) { + MS_EXCEPTION_IF_NULL(me_tensors[index]); + MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); + //定义变量shape为index对应位tensors的shape + //将shape一个个写入shape_str中 + auto shape = me_tensors[index]->shape(); + std::string shape_str; + for (size_t i = 0; i < shape.size(); i++) { + shape_str += std::to_string(shape[i]); + shape_str += " "; + } + //输出消息日志 "me_tensor "[*此处为index(位数)]" 's shape is:"[*此处为index对应位的tensors的shape] + //输出消息日志"me_tensor "[*此处为index(位数)]" 's type is: "[*此处为index对应位的tensors的type] + MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; + MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); + //定义ge_tensor_ptr + auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); + //如果ge_tensor_ptr为空，则执行对应操作 + //否则，输出错误日志 "Convert me_tensor "[*此处为index(位数)]" to Ge Tensor failed!" + //清空ge_tensors并返回 + //这一步为检验此前过程是否成功 + if (ge_tensor_ptr != nullptr) { + ge_tensors.emplace_back(ge_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; + ge_tensors.clear(); + return ge_tensors; + } + } + return ge_tensors; +} + +GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr &tensor, const std::string &format) { + // get tensor data type size + //获取变量tensors的data_type的size属性 + //检验tensor是否为空，并输出相应消息 + MS_EXCEPTION_IF_NULL(tensor); + //获取变量tensors的data_type的size属性 + size_t type_size = GetDataTypeSize(tensor->data_type()); + //判断其是否为空，若为空则输出错误日志"The Me Tensor data type size is wrong, type size is: "[*此处为type_size] + if (type_size == kErrorSize) { + MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; + return nullptr; + } + size_t elements_num = IntToSize(tensor->ElementsNum()); + + // get tensor buff size + //获取变量tensor的data_buff_size属性 + size_t data_buff_size = elements_num * type_size; + //检验其是否为空，若为空，则输出消息日志"The Me Tensor data buff size is 0." + if (data_buff_size == 0) { + MS_LOG(INFO) << "The Me Tensor data buff size is 0."; + } + // create ge tensor + //创建ge_tensor + //将tensors的属性写入到desc中 + auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + //检验其是否为空，若为空，则输出错误日志 "Failed to get Tensor Desc" + //这一步主要检验获取data_type等情况 + if (desc == nullptr) { + MS_LOG(ERROR) << "Failed to get Tensor Desc"; + return nullptr; + } + //将desc和tensors有关属性存入tensors_ptr中 + GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); + //检验其是否为空，若不为空，则输出消息日志"Convert Me Tensor to Ge Tensor success!" + //此处为检验上述过程是否成功 + if (tensor_ptr != nullptr) { + MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; + } + return tensor_ptr; +} +//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors, + const std::vector &request_dims) { + //定义outputs + std::vector outputs; + //在index小于ge_tensors.size的范围内递增循环 + //定义me_tensor_ptr为空 + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = nullptr; + //判断index和request_dims.size的大小 + //若后者较大，则将index对应位置的ge_tensors和request_dims存入me_tensor_ptr + //反之则定义empty_shape并将其与index对应位置的ge_tensors存入me_tensor_ptr + if (index < request_dims.size()) { + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); + } else { + ShapeVector empty_shape; + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); + } + //判断me_tensor_ptr是否为空 + //若不为空，则输出me_tensor_ptr相应内容 + //若为空，则出书错误日志"Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs + //此处为检验上述判断大小部分的功能是否成功实现 + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} +//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors) { + //定义outputs + std::vector outputs; + //在index小于ge_tensors.size的范围内递增循环 + //定义me_tensor_ptr，储存经ConvertGeTensor处理后的index位ge_tensors + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); + //若me_tensor_ptr不为空，则进行相应output处理 + //否则，输出错误日志 "Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs + //主要检验ge_tensors与me_tensors的转化是否成功 + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} +//在TransformUtil区域中创建名为:ConvertGeDataType的函数，返回值类型为自定义MeDataType +//此函数功能为检验type的类型，在type类型为DT_*的情况下返回对应的kNumberType* +//如果变量type不属于DT_*中的任何一种，则返回MeDataType::kTypeUnknown +MeDataType TransformUtil::ConvertGeDataType(const GeDataType &type) { + switch (type) { + case GeDataType::DT_FLOAT16: + return MeDataType::kNumberTypeFloat16; + case GeDataType::DT_FLOAT: + return MeDataType::kNumberTypeFloat32; + case GeDataType::DT_DOUBLE: + return MeDataType::kNumberTypeFloat64; + case GeDataType::DT_INT64: + return MeDataType::kNumberTypeInt64; + case GeDataType::DT_INT32: + return MeDataType::kNumberTypeInt32; + case GeDataType::DT_INT16: + return MeDataType::kNumberTypeInt16; + case GeDataType::DT_INT8: + return MeDataType::kNumberTypeInt8; + case GeDataType::DT_BOOL: + return MeDataType::kNumberTypeBool; + case GeDataType::DT_UINT8: + return MeDataType::kNumberTypeUInt8; + case GeDataType::DT_UINT16: + return MeDataType::kNumberTypeUInt16; + case GeDataType::DT_UINT32: + return MeDataType::kNumberTypeUInt32; + case GeDataType::DT_UINT64: + return MeDataType::kNumberTypeUInt64; + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL_SUB_UINT8: + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_DUAL: + return MeDataType::kTypeUnknown; + default: + return MeDataType::kTypeUnknown; + } +} + +namespace { +//定义名为IsGeShapeCompatible的函数，返回值为bool +bool IsGeShapeCompatible(const GeShape &ge_shape, const ShapeVector &request_dims) { + //输出消息日志，说明当前Getensors的shape属性和我需要的shape属性 + MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); + MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); + + const int GE_DIMS = 4; + std::vector ge_dims = ge_shape.GetDims(); + //判断所需的size与当前size的大小，若前者较大，则输出错误日志 "Request shape's dims count greater than ge shape's" + //并返回false + if (request_dims.size() > ge_dims.size()) { + MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; + return false; + } + + // convert NHWC to NCHW 将NHWC转化为NCHW + //if括号中的内容为判断ge_tensors与re_quest的shape是否相容的条件 + //若符合条件则输出消息日志"Ge tensor shape and request shape is compatible"并返回true + if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && + (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; + } + //令i从0到request_dims.size范围内循环并判断ge_dims[i]与request_dims[i]是否相等 + //若不相等，则输出错误日志"Request shape's dims value not equal to ge shape's"并返回false + std::string::size_type i = 0; + for (; i < request_dims.size(); i++) { + if (ge_dims[i] != request_dims[i]) { + MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; + return false; + } + } + //判断i递增过程中对应的ge_dims[i]是否为1，并输出相应错误日志并返回false + for (; i < ge_dims.size(); i++) { + if (ge_dims[i] != 1) { + MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; + return false; + } + } + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; +} +} // namespace + +//利用变量ge_dims实现变量me_dims向Geshape类型的转换 +GeShape TransformUtil::ConvertMeShape(const ShapeVector &me_dims) { + std::vector ge_dims; + (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); + return GeShape(ge_dims); +} +//利用变量me_dims实现变量ge_shape向ShapeVector类型的转换 +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape) { + ShapeVector me_dims; + std::vector ge_dims = ge_shape.GetDims(); + (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); + return me_dims; +} +//在TransformUtil区域中创建名为ConvertGeShape的函数，返回值类型为自定义类型ShapeVector +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape, const ShapeVector &request_dims) { + vector ret; + //判断ge_shape的对应属性是否为零，若为零则输出错误日志 "GeTensor's shape is scalar"并返回ret + if (ge_shape.GetDimNum() == 0) { + MS_LOG(DEBUG) << "GeTensor's shape is scalar"; + return ret; + } + //判断ge_shape是否可可兼容，若可兼容则用ret储存相应的值，否则输出相应错误日志并将ge_shape进行转换并用ret储存 + if (IsGeShapeCompatible(ge_shape, request_dims) == true) { + ret = request_dims; + } else { + MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; + ret = ConvertGeShape(ge_shape); + } + return ret; +} +//在TransformUtil区域中创建名为GenerateMeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const ShapeVector &me_dims, + const TypeId &me_type) { + MeTensor me_tensor(me_type, me_dims); + + // Get the writable data pointer of the tensor and cast it to its data type + //获取tensors的可写的数据指针并将其改写成其datatype + auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); + size_t me_data_size = static_cast(me_tensor.data().nbytes());//获取tensors的data size属性 + MS_EXCEPTION_IF_NULL(me_data_ptr);//检测me_data_ptr是否为空 + MS_EXCEPTION_IF_NULL(ge_tensor);//检测ge_tensor是否为空 + //判断me_data_size和ge_tensor的size的大小，若前者较小，则输出对应错误日志 + //"ME tensor data size["[*此处为me_data_size]" bytes] is less than GE tensor ["[*此处为ge_tensors的size]" bytes]" + //并返回nullptr + if (me_data_size < ge_tensor->GetSize()) { + MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" + << ge_tensor->GetSize() << " bytes]"; + return nullptr; + } + + // Copy or use the writable data pointer of the ME tensor + //复制或使用me_tensors的可写数据指针 + MS_EXCEPTION_IF_NULL(ge_tensor->GetData());//判断ge_tensors的data是否为空 + //判断ge_tensors的size属性是否为零，若为零，则输出相应错误日志并返回nullptr + if (ge_tensor->GetSize() == 0) { + MS_LOG(ERROR) << "GE tensor data size is zero!"; + return nullptr; + } + + // Use memcpy here, not memcpy_s, just because the size of ge_tensor may be bigger than 2GB + //因为ge_tensor的size可能大于2GB，所以在此处应用memcpy而不是memcpy_s + // which is the size limit of memcpy_s + //2GB为memcpy_s的极限容量 + (void)memcpy(me_data_ptr, ge_tensor->GetData(), ge_tensor->GetSize()); + + return make_shared(me_tensor); +} +//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr &ge_tensor) { + MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape + vector me_dims = ConvertGeShape(ge_shape); + + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype + //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} + +// if request_dims is empty, use ge tensor's shape, otherwise convert to request shape +//若 request_dim为空，则使用ge_tensors的shape，用其他的方式转化成要求的shape +//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const ShapeVector &request_dims) { + MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape + vector me_dims = ConvertGeShape(ge_shape, request_dims); + //输出信息日志 "GE tensor type is [*此处为ge_tensors的datatype]" + MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + // Create a tensor with wanted data type and shape + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype + //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} +//在TransformUtil区域中创建名为PrintGeTensor的函数，返回值类型为std::string +std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { + std::string ret; + //检验变量ge_tensors是否为空，若为空则输出错误日志"Input ge tensor is nullptr"并返回ret + if (ge_tensor == nullptr) { + MS_LOG(ERROR) << "Input ge tensor is nullptr"; + return ret; + } + //ge_tensors不为空执行以下语句 + //输出信息日志"Ge Tensor data type is : [*此处为ge_tensors的data_type]" + //对于ge_trensors的data_type进行比对，若其符合GeDataType::DT_*等一系列的值，则令ret等于对应的值并结束语句 + //若其不符合这一系列的值，则输出错误日志"Unsupported to print type:"[*此处为ge_tensors的data type]" ge tensor";并结束语句 + MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + switch (static_cast(ge_tensor->GetTensorDesc().GetDataType())) { + case GeDataType::DT_UINT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DOUBLE: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_INT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT8: + case GeDataType::DT_DUAL_SUB_UINT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT16: + case GeDataType::DT_BOOL: + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL: + default: + MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) + << " ge tensor"; + break; + } + return ret; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 52b32bc041c040caf2c4df4d9074df87d5877638 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:38:52 +0800 Subject: [PATCH 072/160] ADD file via upload --- .../ccsrc/minddata/mindrecord/include | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include b/mindspore2022/ccsrc/minddata/mindrecord/include new file mode 100644 index 00000000000..056d086da08 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include @@ -0,0 +1,210 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ + +#include +#include +#include +#include +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_error.h" +#include "nlohmann/json.hpp" +#include "./sqlite3.h" +#include "utils/log_adapter.h" + +/* To be used when dlog is ok #include "./slog.h" */ +#ifdef DEBUG +#define MS_ASSERT(f) assert(f) +#else +#define MS_ASSERT(f) ((void)0) +#endif + +namespace mindspore {// +namespace mindrecord { +using json = nlohmann::json; + +//定义int型常量KInt1，KInt2，KInt3，kUnsignedInt4 +const int kInt0 = 0; +const int kInt1 = 1; +const int kInt2 = 2; +const int kInt3 = 3; +const int kUnsignedInt4 = 4; + +enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel }; + +const char kVersion[] = "3.0"; //将3.0赋给字符型数组kVersion[] +const std::vector kSupportedVersion = {"2.0", kVersion}; //将2.0，kVersion赋给vector kSupportedVersion + +enum ShardType { + kNLP = 0, + kCV = 1, +}; + +enum TaskType { + kCommonTask = 0, + kPaddedTask = 1, +}; +enum SamplerType { kCustomTopNSampler, kCustomTopPercentSampler, kSubsetRandomSampler, kPKSampler, kSubsetSampler }; + +enum ShuffleType { kShuffleCategory, kShuffleSample }; + +const double kEpsilon = 1e-7; + +const int kThreadNumber = 14; + +// Shard default parameters Shard默认参数 +const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB +const uint64_t kDefaultPageSize = 1 << 25; // 32MB + +// HeaderSize [16KB, 128MB] +const int kMinHeaderSize = 1 << 14; // 16KB +const int kMaxHeaderSize = 1 << 27; // 128MB + +// PageSize [32KB, 256MB] +const int kMinPageSize = 1 << 15; // 32KB +const int kMaxPageSize = 1 << 28; // 256MB + +// used by value length / schema id length / statistic id length ... 由值长度/架构id长度/统计id长度使用。。。 +const uint64_t kInt64Len = 8; + +// Minimum file size 最小文件大小 +const uint64_t kMinFileSize = kInt64Len; + +const int kMinShardCount = 1; +const int kMaxShardCount = 1000; // write +const int kMaxFileCount = 4096; // read + +const int kMinConsumerCount = 1; +const int kMaxConsumerCount = 128; + +const int kMaxSchemaCount = 1; +const int kMaxThreadCount = 32; +const int kMaxFieldCount = 100; + +// Minimum free disk size 最小可用磁盘大小 +const int kMinFreeDiskSize = 10; // 10M + +// dummy json +const json kDummyId = R"({"id": 0})"_json; + +// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) 将模式中的类型转换为sqlite3中的类型（NULL、INTEGER、REAL、TEXT、BLOB） +const std::unordered_map kDbJsonMap = { + {"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"}, + {"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"}, + {"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"}, + {"bytes", "BLOB"}}; + +const char kPoint = '.'; + +const char kPathSeparator = +#if defined(_WIN32) || defined(_WIN64) + '\\'; +#else + '/'; +#endif + +// field type used by check schema validation 检查架构验证使用的字段类型 +const std::set kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"}; + +// can be searched field list 可搜索字段列表 +const std::set kScalarFieldTypeSet = {"string", "int32", "int64", "float32", "float64"}; + +// number field list 数字字段列表 +const std::set kNumberFieldTypeSet = {"int32", "int64", "float32", "float64"}; + +const std::unordered_map kTypesMap = { + {"bool", "int32"}, {"int8", "int32"}, {"uint8", "bytes"}, {"int16", "int32"}, + {"uint16", "int32"}, {"int32", "int32"}, {"uint32", "int64"}, {"int64", "int64"}, + {"float16", "float32"}, {"float32", "float32"}, {"float64", "float64"}, {"string", "string"}}; + +/// \brief the max number of samples to enable lazy load 启用延迟加载的最大样本数 +const uint32_t LAZY_LOAD_THRESHOLD = 5000000; + +/// \brief split a string using a character 使用字符拆分字符串 +/// \param[in] field target string 目标字符串 +/// \param[in] separator a character for splitting 用于拆分的字符 +/// \return vector type result 矢量类型结果 +std::vector StringSplit(const std::string &field, char separator); + +/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' 验证字段名由“0-9”或“a-z”或“a-z”或“_”或“-”组成 +/// \param[in] str target string 目标字符串 +/// \return +bool ValidateFieldName(const std::string &str); + +/// \brief get the filename by the path 通过路径获取文件名 +/// \param s file path 文件路径 +/// \param fn_ptr shared ptr of file name 文件名的共享指针 +/// \return Status +Status GetFileName(const std::string &path, std::shared_ptr *fn_ptr); + +/// \brief get parent dir 获取父目录 +/// \param path file path 文件路径 +/// \param pd_ptr shared ptr of parent path 父路径的共享指针 +/// \return Status +Status GetParentDir(const std::string &path, std::shared_ptr *pd_ptr); + +bool CheckIsValidUtf8(const std::string &str); + +/// \brief judge if a path is legal file 判断路径是否为合法文件 +/// \param path file path 文件路径 +/// \return Whether the path is legal or not 路径是否合法 +bool IsLegalFile(const std::string &path); + +enum DiskSizeType { kTotalSize = 0, kFreeSize }; + +/// \brief get the free space about the disk 获取磁盘的可用空间 +/// \param str_dir file path 文件路径 +/// \param disk_type: kTotalSize / kFreeSize +/// \param size: shared ptr of size in Megabytes 以MB为单位的共享指针 +/// \return Status +Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr *size); + +/// \brief get the max hardware concurrency //获取最大硬件并发 +/// \return max concurrency 最大并发数 +uint32_t GetMaxThreadNum(); + +/// \brief get absolute path of all mindrecord files 获取所有mindrecord文件的绝对路径 +/// \param path path to one fo mindrecord files 一个fo-mindrecord文件的路径 +/// \param addresses relative path of all mindrecord files 所有思维记录文件的相对路径 +/// \param ds shared ptr of vector of absolute path 绝对路径矢量的共享指针 +/// \return Status +Status GetDatasetFiles(const std::string &path, const json &addresses, std::shared_ptr> *ds); +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -- 2.34.1 From 744689c7da451131238c8374b1609831158a6e82 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:40:19 +0800 Subject: [PATCH 073/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include' --- .../ccsrc/minddata/mindrecord/include | 210 ------------------ 1 file changed, 210 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include b/mindspore2022/ccsrc/minddata/mindrecord/include deleted file mode 100644 index 056d086da08..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ - -#include -#include -#include -#include -#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "minddata/mindrecord/include/shard_error.h" -#include "nlohmann/json.hpp" -#include "./sqlite3.h" -#include "utils/log_adapter.h" - -/* To be used when dlog is ok #include "./slog.h" */ -#ifdef DEBUG -#define MS_ASSERT(f) assert(f) -#else -#define MS_ASSERT(f) ((void)0) -#endif - -namespace mindspore {// -namespace mindrecord { -using json = nlohmann::json; - -//定义int型常量KInt1，KInt2，KInt3，kUnsignedInt4 -const int kInt0 = 0; -const int kInt1 = 1; -const int kInt2 = 2; -const int kInt3 = 3; -const int kUnsignedInt4 = 4; - -enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel }; - -const char kVersion[] = "3.0"; //将3.0赋给字符型数组kVersion[] -const std::vector kSupportedVersion = {"2.0", kVersion}; //将2.0，kVersion赋给vector kSupportedVersion - -enum ShardType { - kNLP = 0, - kCV = 1, -}; - -enum TaskType { - kCommonTask = 0, - kPaddedTask = 1, -}; -enum SamplerType { kCustomTopNSampler, kCustomTopPercentSampler, kSubsetRandomSampler, kPKSampler, kSubsetSampler }; - -enum ShuffleType { kShuffleCategory, kShuffleSample }; - -const double kEpsilon = 1e-7; - -const int kThreadNumber = 14; - -// Shard default parameters Shard默认参数 -const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB -const uint64_t kDefaultPageSize = 1 << 25; // 32MB - -// HeaderSize [16KB, 128MB] -const int kMinHeaderSize = 1 << 14; // 16KB -const int kMaxHeaderSize = 1 << 27; // 128MB - -// PageSize [32KB, 256MB] -const int kMinPageSize = 1 << 15; // 32KB -const int kMaxPageSize = 1 << 28; // 256MB - -// used by value length / schema id length / statistic id length ... 由值长度/架构id长度/统计id长度使用。。。 -const uint64_t kInt64Len = 8; - -// Minimum file size 最小文件大小 -const uint64_t kMinFileSize = kInt64Len; - -const int kMinShardCount = 1; -const int kMaxShardCount = 1000; // write -const int kMaxFileCount = 4096; // read - -const int kMinConsumerCount = 1; -const int kMaxConsumerCount = 128; - -const int kMaxSchemaCount = 1; -const int kMaxThreadCount = 32; -const int kMaxFieldCount = 100; - -// Minimum free disk size 最小可用磁盘大小 -const int kMinFreeDiskSize = 10; // 10M - -// dummy json -const json kDummyId = R"({"id": 0})"_json; - -// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) 将模式中的类型转换为sqlite3中的类型（NULL、INTEGER、REAL、TEXT、BLOB） -const std::unordered_map kDbJsonMap = { - {"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"}, - {"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"}, - {"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"}, - {"bytes", "BLOB"}}; - -const char kPoint = '.'; - -const char kPathSeparator = -#if defined(_WIN32) || defined(_WIN64) - '\\'; -#else - '/'; -#endif - -// field type used by check schema validation 检查架构验证使用的字段类型 -const std::set kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"}; - -// can be searched field list 可搜索字段列表 -const std::set kScalarFieldTypeSet = {"string", "int32", "int64", "float32", "float64"}; - -// number field list 数字字段列表 -const std::set kNumberFieldTypeSet = {"int32", "int64", "float32", "float64"}; - -const std::unordered_map kTypesMap = { - {"bool", "int32"}, {"int8", "int32"}, {"uint8", "bytes"}, {"int16", "int32"}, - {"uint16", "int32"}, {"int32", "int32"}, {"uint32", "int64"}, {"int64", "int64"}, - {"float16", "float32"}, {"float32", "float32"}, {"float64", "float64"}, {"string", "string"}}; - -/// \brief the max number of samples to enable lazy load 启用延迟加载的最大样本数 -const uint32_t LAZY_LOAD_THRESHOLD = 5000000; - -/// \brief split a string using a character 使用字符拆分字符串 -/// \param[in] field target string 目标字符串 -/// \param[in] separator a character for splitting 用于拆分的字符 -/// \return vector type result 矢量类型结果 -std::vector StringSplit(const std::string &field, char separator); - -/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' 验证字段名由“0-9”或“a-z”或“a-z”或“_”或“-”组成 -/// \param[in] str target string 目标字符串 -/// \return -bool ValidateFieldName(const std::string &str); - -/// \brief get the filename by the path 通过路径获取文件名 -/// \param s file path 文件路径 -/// \param fn_ptr shared ptr of file name 文件名的共享指针 -/// \return Status -Status GetFileName(const std::string &path, std::shared_ptr *fn_ptr); - -/// \brief get parent dir 获取父目录 -/// \param path file path 文件路径 -/// \param pd_ptr shared ptr of parent path 父路径的共享指针 -/// \return Status -Status GetParentDir(const std::string &path, std::shared_ptr *pd_ptr); - -bool CheckIsValidUtf8(const std::string &str); - -/// \brief judge if a path is legal file 判断路径是否为合法文件 -/// \param path file path 文件路径 -/// \return Whether the path is legal or not 路径是否合法 -bool IsLegalFile(const std::string &path); - -enum DiskSizeType { kTotalSize = 0, kFreeSize }; - -/// \brief get the free space about the disk 获取磁盘的可用空间 -/// \param str_dir file path 文件路径 -/// \param disk_type: kTotalSize / kFreeSize -/// \param size: shared ptr of size in Megabytes 以MB为单位的共享指针 -/// \return Status -Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr *size); - -/// \brief get the max hardware concurrency //获取最大硬件并发 -/// \return max concurrency 最大并发数 -uint32_t GetMaxThreadNum(); - -/// \brief get absolute path of all mindrecord files 获取所有mindrecord文件的绝对路径 -/// \param path path to one fo mindrecord files 一个fo-mindrecord文件的路径 -/// \param addresses relative path of all mindrecord files 所有思维记录文件的相对路径 -/// \param ds shared ptr of vector of absolute path 绝对路径矢量的共享指针 -/// \return Status -Status GetDatasetFiles(const std::string &path, const json &addresses, std::shared_ptr> *ds); -} // namespace mindrecord -} // namespace mindspore - -#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -- 2.34.1 From 1d68c9de6c260bba4cc5080a194932248ac834f6 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:41:57 +0800 Subject: [PATCH 074/160] ADD file via upload --- .../ccsrc/Fminddata/mindrecord/include | 549 ++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100644 mindspore2022/ccsrc/Fminddata/mindrecord/include diff --git a/mindspore2022/ccsrc/Fminddata/mindrecord/include b/mindspore2022/ccsrc/Fminddata/mindrecord/include new file mode 100644 index 00000000000..eda89bd3759 --- /dev/null +++ b/mindspore2022/ccsrc/Fminddata/mindrecord/include @@ -0,0 +1,549 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/util.h" + +#include +#include + +#include "securec/include/securec.h" +#include "include/common/utils/convert_utils.h" +#include "include/common/utils/utils.h" + +namespace mindspore { +namespace transform { +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; + +const size_t kErrorSize = 0; +//在TransformUtil区域中创建名为ConvertIntToList的函数，返回值类型为vector，用于检测size的长度。 +//若长度小于等于零则弹出MS_LOG警告日志 "size <= 0"并直接返回list +//若长度大于零则用list对data中的内容进行push_back操作 +vector TransformUtil::ConvertIntToList(int64_t data, int size) { + vector list{}; + if (size <= 0) { + MS_LOG(WARNING) << "size <= 0"; + return list; + } + for (int i = 0; i < size; ++i) { + list.push_back(data); + } + return list; +} + +static std::map datatype_trans_map = { + {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, + {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, + {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, + {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, + {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, + {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; +//在TransformUtil区域中创建名为ConvertDataType的函数，返回值类型为自定义GeDataType +//输出MS_LOG调试日志"Convert me data type: "[*此处为对type进行TypeIdLabel(type)的结果]ge data type" +//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 +//若不一致，则返回datatype_trans_map[type] +//若一致则返回DT_UNDEFINED +GeDataType TransformUtil::ConvertDataType(const MeDataType &type) { + MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; + if (datatype_trans_map.find(type) != datatype_trans_map.end()) { + return datatype_trans_map[type]; + } else { + return GeDataType::DT_UNDEFINED; + } +} + +static std::map datatype_size_map = { + {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float + {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, + {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, + {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, + {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, + {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; +//在TransformUtil区域中创建名为GetDataTypeSize的函数，返回值类型为自定义size_t +//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 +//若不一致，则返回datatype_trans_map[type] +//若一致，则输出错误日志 "Illegal tensor data type!"并返回kErrorSize +size_t TransformUtil::GetDataTypeSize(const MeDataType &type) { + if (datatype_size_map.find(type) != datatype_size_map.end()) { + return datatype_size_map[type]; + } else { + MS_LOG(ERROR) << "Illegal tensor data type!"; + return kErrorSize; + } +} +//在TransformUtil区域中创建名为:ConvertFormat的函数，返回值类型为自定义GeFormat +//判断format与 kOpFormat_NCHW等kOpFormat_*是否相等，若相等，则返回 kOpFormat_NCHW等kOpFormat_* +//若与一系列kOpFormat_*都不相等，则输出错误日志 "Illegal tensor data format: ("[*此处为format变量]"). Use ND format instead." +//并返回FORMAT_ND +GeFormat TransformUtil::ConvertFormat(const string &format) { + if (format == kOpFormat_NCHW) { + return GeFormat::FORMAT_NCHW; + } else if (format == kOpFormat_NDHWC) { + return GeFormat::FORMAT_NDHWC; + } else if (format == kOpFormat_NCDHW) { + return GeFormat::FORMAT_NCDHW; + } else if (format == kOpFormat_DHWNC) { + return GeFormat::FORMAT_DHWNC; + } else if (format == kOpFormat_DHWCN) { + return GeFormat::FORMAT_DHWCN; + } else if (format == kOpFormat_NC1HWC0) { + return GeFormat::FORMAT_NC1HWC0; + } else if (format == kOpFormat_NHWC) { + return GeFormat::FORMAT_NHWC; + } else if (format == kOpFormat_HWCN) { + return GeFormat::FORMAT_HWCN; + } else if (format == kOpFormat_ND) { + return GeFormat::FORMAT_ND; + } else { + MS_LOG(ERROR) << "Illegal tensor data format: (" << format << "). Use ND format instead."; + return GeFormat::FORMAT_ND; + } +} +//创建名为IntegerCastFunc的函数，返回值类型为自定义static int64_t +//返回执行static_cast()操作后的size_t的temp +static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } + +//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr TransformUtil::GetGeTensorDesc(const ShapeVector &me_shape, const MeDataType &me_type, + const std::string &format) { + // convert me shape to ge shape + //将me_shape转化为ge_shape + //定义ge_shape + std::vector ge_shape; + //若me_shape.size()=1，则对其第[0]位进行push_back操作 + //否则，则根据me_shape的size属性设置ge_shape的size属性 + if (me_shape.size() == 1) { + ge_shape.push_back(static_cast(me_shape[0])); + } else { + ge_shape.resize(me_shape.size()); + (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); + } + //若shape.GetDimNum()=0，则输出信息日志"The dims size of Ge tensor is zero" + GeShape shape(ge_shape); + if (shape.GetDimNum() == 0) { + MS_LOG(INFO) << "The dims size of Ge tensor is zero"; + } + // convert me format to ge format + //将me_format转化为ge_format + //若ge_format符合相应条件则输出信息日志 "Set ND data format" + GeFormat ge_format = ConvertFormat(format); + if (ge_format == GeFormat::FORMAT_ND) { + MS_LOG(INFO) << "Set ND data format"; + } + // convert me datatype to ge datatype + //将me_datatype转化为ge_datatype + //若data_type符合相应条件则输出错误日志"undefined data type :"[*此处为me_type] + //并返回 nullptr + GeDataType data_type = ConvertDataType(me_type); + if (data_type == GeDataType::DT_UNDEFINED) { + MS_LOG(ERROR) << "undefined data type :" << me_type; + return nullptr; + } + //进行make_shared操作，若desc变量为空则输出错误日志 "Create GeTensorDesc failed!" + //并返回 nullptr + //若不为空则输出信息日志"SetRealDimCnt is :"[*此处为 me_shape.size()] + //并对desc进行SetRealDimCnt操作 + //并返回desc + //这一步在检验转换是否成功并且转换后对象的size属性正确 + auto desc = std::make_shared(shape, ge_format, data_type); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create GeTensorDesc failed!"; + return nullptr; + } + MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); + desc->SetRealDimCnt(SizeToInt(me_shape.size())); + return desc; +} + +// if failed, return empty vector. +//如果失败了，则返回空的vector +//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr +std::vector TransformUtil::ConvertInputTensors(const std::vector &me_tensors, + const std::string &format) { + //定义ge_tensors + std::vector ge_tensors; + //在index小于me_tensors.size的范围内递增循环 + //输出消息日志 "me_tensor "[*此处为index(tensors的位数)]" 's data size is: "[*此处为index对应位的tensors的size] + for (size_t index = 0; index < me_tensors.size(); index++) { + MS_EXCEPTION_IF_NULL(me_tensors[index]); + MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); + //定义变量shape为index对应位tensors的shape + //将shape一个个写入shape_str中 + auto shape = me_tensors[index]->shape(); + std::string shape_str; + for (size_t i = 0; i < shape.size(); i++) { + shape_str += std::to_string(shape[i]); + shape_str += " "; + } + //输出消息日志 "me_tensor "[*此处为index(位数)]" 's shape is:"[*此处为index对应位的tensors的shape] + //输出消息日志"me_tensor "[*此处为index(位数)]" 's type is: "[*此处为index对应位的tensors的type] + MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; + MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); + //定义ge_tensor_ptr + auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); + //如果ge_tensor_ptr为空，则执行对应操作 + //否则，输出错误日志 "Convert me_tensor "[*此处为index(位数)]" to Ge Tensor failed!" + //清空ge_tensors并返回 + //这一步为检验此前过程是否成功 + if (ge_tensor_ptr != nullptr) { + ge_tensors.emplace_back(ge_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; + ge_tensors.clear(); + return ge_tensors; + } + } + return ge_tensors; +} + +GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr &tensor, const std::string &format) { + // get tensor data type size + //获取变量tensors的data_type的size属性 + //检验tensor是否为空，并输出相应消息 + MS_EXCEPTION_IF_NULL(tensor); + //获取变量tensors的data_type的size属性 + size_t type_size = GetDataTypeSize(tensor->data_type()); + //判断其是否为空，若为空则输出错误日志"The Me Tensor data type size is wrong, type size is: "[*此处为type_size] + if (type_size == kErrorSize) { + MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; + return nullptr; + } + size_t elements_num = IntToSize(tensor->ElementsNum()); + + // get tensor buff size + //获取变量tensor的data_buff_size属性 + size_t data_buff_size = elements_num * type_size; + //检验其是否为空，若为空，则输出消息日志"The Me Tensor data buff size is 0." + if (data_buff_size == 0) { + MS_LOG(INFO) << "The Me Tensor data buff size is 0."; + } + // create ge tensor + //创建ge_tensor + //将tensors的属性写入到desc中 + auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + //检验其是否为空，若为空，则输出错误日志 "Failed to get Tensor Desc" + //这一步主要检验获取data_type等情况 + if (desc == nullptr) { + MS_LOG(ERROR) << "Failed to get Tensor Desc"; + return nullptr; + } + //将desc和tensors有关属性存入tensors_ptr中 + GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); + //检验其是否为空，若不为空，则输出消息日志"Convert Me Tensor to Ge Tensor success!" + //此处为检验上述过程是否成功 + if (tensor_ptr != nullptr) { + MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; + } + return tensor_ptr; +} +//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors, + const std::vector &request_dims) { + //定义outputs + std::vector outputs; + //在index小于ge_tensors.size的范围内递增循环 + //定义me_tensor_ptr为空 + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = nullptr; + //判断index和request_dims.size的大小 + //若后者较大，则将index对应位置的ge_tensors和request_dims存入me_tensor_ptr + //反之则定义empty_shape并将其与index对应位置的ge_tensors存入me_tensor_ptr + if (index < request_dims.size()) { + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); + } else { + ShapeVector empty_shape; + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); + } + //判断me_tensor_ptr是否为空 + //若不为空，则输出me_tensor_ptr相应内容 + //若为空，则出书错误日志"Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs + //此处为检验上述判断大小部分的功能是否成功实现 + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} +//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors) { + //定义outputs + std::vector outputs; + //在index小于ge_tensors.size的范围内递增循环 + //定义me_tensor_ptr，储存经ConvertGeTensor处理后的index位ge_tensors + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); + //若me_tensor_ptr不为空，则进行相应output处理 + //否则，输出错误日志 "Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs + //主要检验ge_tensors与me_tensors的转化是否成功 + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} +//在TransformUtil区域中创建名为:ConvertGeDataType的函数，返回值类型为自定义MeDataType +//此函数功能为检验type的类型，在type类型为DT_*的情况下返回对应的kNumberType* +//如果变量type不属于DT_*中的任何一种，则返回MeDataType::kTypeUnknown +MeDataType TransformUtil::ConvertGeDataType(const GeDataType &type) { + switch (type) { + case GeDataType::DT_FLOAT16: + return MeDataType::kNumberTypeFloat16; + case GeDataType::DT_FLOAT: + return MeDataType::kNumberTypeFloat32; + case GeDataType::DT_DOUBLE: + return MeDataType::kNumberTypeFloat64; + case GeDataType::DT_INT64: + return MeDataType::kNumberTypeInt64; + case GeDataType::DT_INT32: + return MeDataType::kNumberTypeInt32; + case GeDataType::DT_INT16: + return MeDataType::kNumberTypeInt16; + case GeDataType::DT_INT8: + return MeDataType::kNumberTypeInt8; + case GeDataType::DT_BOOL: + return MeDataType::kNumberTypeBool; + case GeDataType::DT_UINT8: + return MeDataType::kNumberTypeUInt8; + case GeDataType::DT_UINT16: + return MeDataType::kNumberTypeUInt16; + case GeDataType::DT_UINT32: + return MeDataType::kNumberTypeUInt32; + case GeDataType::DT_UINT64: + return MeDataType::kNumberTypeUInt64; + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL_SUB_UINT8: + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_DUAL: + return MeDataType::kTypeUnknown; + default: + return MeDataType::kTypeUnknown; + } +} + +namespace { +//定义名为IsGeShapeCompatible的函数，返回值为bool +bool IsGeShapeCompatible(const GeShape &ge_shape, const ShapeVector &request_dims) { + //输出消息日志，说明当前Getensors的shape属性和我需要的shape属性 + MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); + MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); + + const int GE_DIMS = 4; + std::vector ge_dims = ge_shape.GetDims(); + //判断所需的size与当前size的大小，若前者较大，则输出错误日志 "Request shape's dims count greater than ge shape's" + //并返回false + if (request_dims.size() > ge_dims.size()) { + MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; + return false; + } + + // convert NHWC to NCHW 将NHWC转化为NCHW + //if括号中的内容为判断ge_tensors与re_quest的shape是否相容的条件 + //若符合条件则输出消息日志"Ge tensor shape and request shape is compatible"并返回true + if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && + (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; + } + //令i从0到request_dims.size范围内循环并判断ge_dims[i]与request_dims[i]是否相等 + //若不相等，则输出错误日志"Request shape's dims value not equal to ge shape's"并返回false + std::string::size_type i = 0; + for (; i < request_dims.size(); i++) { + if (ge_dims[i] != request_dims[i]) { + MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; + return false; + } + } + //判断i递增过程中对应的ge_dims[i]是否为1，并输出相应错误日志并返回false + for (; i < ge_dims.size(); i++) { + if (ge_dims[i] != 1) { + MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; + return false; + } + } + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; +} +} // namespace + +//利用变量ge_dims实现变量me_dims向Geshape类型的转换 +GeShape TransformUtil::ConvertMeShape(const ShapeVector &me_dims) { + std::vector ge_dims; + (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); + return GeShape(ge_dims); +} +//利用变量me_dims实现变量ge_shape向ShapeVector类型的转换 +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape) { + ShapeVector me_dims; + std::vector ge_dims = ge_shape.GetDims(); + (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); + return me_dims; +} +//在TransformUtil区域中创建名为ConvertGeShape的函数，返回值类型为自定义类型ShapeVector +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape, const ShapeVector &request_dims) { + vector ret; + //判断ge_shape的对应属性是否为零，若为零则输出错误日志 "GeTensor's shape is scalar"并返回ret + if (ge_shape.GetDimNum() == 0) { + MS_LOG(DEBUG) << "GeTensor's shape is scalar"; + return ret; + } + //判断ge_shape是否可可兼容，若可兼容则用ret储存相应的值，否则输出相应错误日志并将ge_shape进行转换并用ret储存 + if (IsGeShapeCompatible(ge_shape, request_dims) == true) { + ret = request_dims; + } else { + MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; + ret = ConvertGeShape(ge_shape); + } + return ret; +} +//在TransformUtil区域中创建名为GenerateMeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const ShapeVector &me_dims, + const TypeId &me_type) { + MeTensor me_tensor(me_type, me_dims); + + // Get the writable data pointer of the tensor and cast it to its data type + //获取tensors的可写的数据指针并将其改写成其datatype + auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); + size_t me_data_size = static_cast(me_tensor.data().nbytes());//获取tensors的data size属性 + MS_EXCEPTION_IF_NULL(me_data_ptr);//检测me_data_ptr是否为空 + MS_EXCEPTION_IF_NULL(ge_tensor);//检测ge_tensor是否为空 + //判断me_data_size和ge_tensor的size的大小，若前者较小，则输出对应错误日志 + //"ME tensor data size["[*此处为me_data_size]" bytes] is less than GE tensor ["[*此处为ge_tensors的size]" bytes]" + //并返回nullptr + if (me_data_size < ge_tensor->GetSize()) { + MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" + << ge_tensor->GetSize() << " bytes]"; + return nullptr; + } + + // Copy or use the writable data pointer of the ME tensor + //复制或使用me_tensors的可写数据指针 + MS_EXCEPTION_IF_NULL(ge_tensor->GetData());//判断ge_tensors的data是否为空 + //判断ge_tensors的size属性是否为零，若为零，则输出相应错误日志并返回nullptr + if (ge_tensor->GetSize() == 0) { + MS_LOG(ERROR) << "GE tensor data size is zero!"; + return nullptr; + } + + // Use memcpy here, not memcpy_s, just because the size of ge_tensor may be bigger than 2GB + //因为ge_tensor的size可能大于2GB，所以在此处应用memcpy而不是memcpy_s + // which is the size limit of memcpy_s + //2GB为memcpy_s的极限容量 + (void)memcpy(me_data_ptr, ge_tensor->GetData(), ge_tensor->GetSize()); + + return make_shared(me_tensor); +} +//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr &ge_tensor) { + MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape + vector me_dims = ConvertGeShape(ge_shape); + + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype + //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} + +// if request_dims is empty, use ge tensor's shape, otherwise convert to request shape +//若 request_dim为空，则使用ge_tensors的shape，用其他的方式转化成要求的shape +//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const ShapeVector &request_dims) { + MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape + vector me_dims = ConvertGeShape(ge_shape, request_dims); + //输出信息日志 "GE tensor type is [*此处为ge_tensors的datatype]" + MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + // Create a tensor with wanted data type and shape + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype + //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} +//在TransformUtil区域中创建名为PrintGeTensor的函数，返回值类型为std::string +std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { + std::string ret; + //检验变量ge_tensors是否为空，若为空则输出错误日志"Input ge tensor is nullptr"并返回ret + if (ge_tensor == nullptr) { + MS_LOG(ERROR) << "Input ge tensor is nullptr"; + return ret; + } + //ge_tensors不为空执行以下语句 + //输出信息日志"Ge Tensor data type is : [*此处为ge_tensors的data_type]" + //对于ge_trensors的data_type进行比对，若其符合GeDataType::DT_*等一系列的值，则令ret等于对应的值并结束语句 + //若其不符合这一系列的值，则输出错误日志"Unsupported to print type:"[*此处为ge_tensors的data type]" ge tensor";并结束语句 + MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + switch (static_cast(ge_tensor->GetTensorDesc().GetDataType())) { + case GeDataType::DT_UINT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DOUBLE: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_INT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT8: + case GeDataType::DT_DUAL_SUB_UINT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT16: + case GeDataType::DT_BOOL: + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL: + default: + MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) + << " ge tensor"; + break; + } + return ret; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From de6dc1ba816419afe2ce41fcfaea465b28d7d254 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:42:52 +0800 Subject: [PATCH 075/160] Delete 'mindspore2022/ccsrc/Fminddata/mindrecord/include' --- .../ccsrc/Fminddata/mindrecord/include | 549 ------------------ 1 file changed, 549 deletions(-) delete mode 100644 mindspore2022/ccsrc/Fminddata/mindrecord/include diff --git a/mindspore2022/ccsrc/Fminddata/mindrecord/include b/mindspore2022/ccsrc/Fminddata/mindrecord/include deleted file mode 100644 index eda89bd3759..00000000000 --- a/mindspore2022/ccsrc/Fminddata/mindrecord/include +++ /dev/null @@ -1,549 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "include/transform/graph_ir/util.h" - -#include -#include - -#include "securec/include/securec.h" -#include "include/common/utils/convert_utils.h" -#include "include/common/utils/utils.h" - -namespace mindspore { -namespace transform { -using std::make_shared; -using std::shared_ptr; -using std::string; -using std::vector; - -const size_t kErrorSize = 0; -//在TransformUtil区域中创建名为ConvertIntToList的函数，返回值类型为vector，用于检测size的长度。 -//若长度小于等于零则弹出MS_LOG警告日志 "size <= 0"并直接返回list -//若长度大于零则用list对data中的内容进行push_back操作 -vector TransformUtil::ConvertIntToList(int64_t data, int size) { - vector list{}; - if (size <= 0) { - MS_LOG(WARNING) << "size <= 0"; - return list; - } - for (int i = 0; i < size; ++i) { - list.push_back(data); - } - return list; -} - -static std::map datatype_trans_map = { - {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, - {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, - {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, - {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, - {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, - {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; -//在TransformUtil区域中创建名为ConvertDataType的函数，返回值类型为自定义GeDataType -//输出MS_LOG调试日志"Convert me data type: "[*此处为对type进行TypeIdLabel(type)的结果]ge data type" -//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 -//若不一致，则返回datatype_trans_map[type] -//若一致则返回DT_UNDEFINED -GeDataType TransformUtil::ConvertDataType(const MeDataType &type) { - MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; - if (datatype_trans_map.find(type) != datatype_trans_map.end()) { - return datatype_trans_map[type]; - } else { - return GeDataType::DT_UNDEFINED; - } -} - -static std::map datatype_size_map = { - {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float - {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, - {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, - {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, - {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, - {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; -//在TransformUtil区域中创建名为GetDataTypeSize的函数，返回值类型为自定义size_t -//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 -//若不一致，则返回datatype_trans_map[type] -//若一致，则输出错误日志 "Illegal tensor data type!"并返回kErrorSize -size_t TransformUtil::GetDataTypeSize(const MeDataType &type) { - if (datatype_size_map.find(type) != datatype_size_map.end()) { - return datatype_size_map[type]; - } else { - MS_LOG(ERROR) << "Illegal tensor data type!"; - return kErrorSize; - } -} -//在TransformUtil区域中创建名为:ConvertFormat的函数，返回值类型为自定义GeFormat -//判断format与 kOpFormat_NCHW等kOpFormat_*是否相等，若相等，则返回 kOpFormat_NCHW等kOpFormat_* -//若与一系列kOpFormat_*都不相等，则输出错误日志 "Illegal tensor data format: ("[*此处为format变量]"). Use ND format instead." -//并返回FORMAT_ND -GeFormat TransformUtil::ConvertFormat(const string &format) { - if (format == kOpFormat_NCHW) { - return GeFormat::FORMAT_NCHW; - } else if (format == kOpFormat_NDHWC) { - return GeFormat::FORMAT_NDHWC; - } else if (format == kOpFormat_NCDHW) { - return GeFormat::FORMAT_NCDHW; - } else if (format == kOpFormat_DHWNC) { - return GeFormat::FORMAT_DHWNC; - } else if (format == kOpFormat_DHWCN) { - return GeFormat::FORMAT_DHWCN; - } else if (format == kOpFormat_NC1HWC0) { - return GeFormat::FORMAT_NC1HWC0; - } else if (format == kOpFormat_NHWC) { - return GeFormat::FORMAT_NHWC; - } else if (format == kOpFormat_HWCN) { - return GeFormat::FORMAT_HWCN; - } else if (format == kOpFormat_ND) { - return GeFormat::FORMAT_ND; - } else { - MS_LOG(ERROR) << "Illegal tensor data format: (" << format << "). Use ND format instead."; - return GeFormat::FORMAT_ND; - } -} -//创建名为IntegerCastFunc的函数，返回值类型为自定义static int64_t -//返回执行static_cast()操作后的size_t的temp -static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } - -//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr TransformUtil::GetGeTensorDesc(const ShapeVector &me_shape, const MeDataType &me_type, - const std::string &format) { - // convert me shape to ge shape - //将me_shape转化为ge_shape - //定义ge_shape - std::vector ge_shape; - //若me_shape.size()=1，则对其第[0]位进行push_back操作 - //否则，则根据me_shape的size属性设置ge_shape的size属性 - if (me_shape.size() == 1) { - ge_shape.push_back(static_cast(me_shape[0])); - } else { - ge_shape.resize(me_shape.size()); - (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); - } - //若shape.GetDimNum()=0，则输出信息日志"The dims size of Ge tensor is zero" - GeShape shape(ge_shape); - if (shape.GetDimNum() == 0) { - MS_LOG(INFO) << "The dims size of Ge tensor is zero"; - } - // convert me format to ge format - //将me_format转化为ge_format - //若ge_format符合相应条件则输出信息日志 "Set ND data format" - GeFormat ge_format = ConvertFormat(format); - if (ge_format == GeFormat::FORMAT_ND) { - MS_LOG(INFO) << "Set ND data format"; - } - // convert me datatype to ge datatype - //将me_datatype转化为ge_datatype - //若data_type符合相应条件则输出错误日志"undefined data type :"[*此处为me_type] - //并返回 nullptr - GeDataType data_type = ConvertDataType(me_type); - if (data_type == GeDataType::DT_UNDEFINED) { - MS_LOG(ERROR) << "undefined data type :" << me_type; - return nullptr; - } - //进行make_shared操作，若desc变量为空则输出错误日志 "Create GeTensorDesc failed!" - //并返回 nullptr - //若不为空则输出信息日志"SetRealDimCnt is :"[*此处为 me_shape.size()] - //并对desc进行SetRealDimCnt操作 - //并返回desc - //这一步在检验转换是否成功并且转换后对象的size属性正确 - auto desc = std::make_shared(shape, ge_format, data_type); - if (desc == nullptr) { - MS_LOG(ERROR) << "Create GeTensorDesc failed!"; - return nullptr; - } - MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); - desc->SetRealDimCnt(SizeToInt(me_shape.size())); - return desc; -} - -// if failed, return empty vector. -//如果失败了，则返回空的vector -//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr -std::vector TransformUtil::ConvertInputTensors(const std::vector &me_tensors, - const std::string &format) { - //定义ge_tensors - std::vector ge_tensors; - //在index小于me_tensors.size的范围内递增循环 - //输出消息日志 "me_tensor "[*此处为index(tensors的位数)]" 's data size is: "[*此处为index对应位的tensors的size] - for (size_t index = 0; index < me_tensors.size(); index++) { - MS_EXCEPTION_IF_NULL(me_tensors[index]); - MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); - //定义变量shape为index对应位tensors的shape - //将shape一个个写入shape_str中 - auto shape = me_tensors[index]->shape(); - std::string shape_str; - for (size_t i = 0; i < shape.size(); i++) { - shape_str += std::to_string(shape[i]); - shape_str += " "; - } - //输出消息日志 "me_tensor "[*此处为index(位数)]" 's shape is:"[*此处为index对应位的tensors的shape] - //输出消息日志"me_tensor "[*此处为index(位数)]" 's type is: "[*此处为index对应位的tensors的type] - MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; - MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); - //定义ge_tensor_ptr - auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); - //如果ge_tensor_ptr为空，则执行对应操作 - //否则，输出错误日志 "Convert me_tensor "[*此处为index(位数)]" to Ge Tensor failed!" - //清空ge_tensors并返回 - //这一步为检验此前过程是否成功 - if (ge_tensor_ptr != nullptr) { - ge_tensors.emplace_back(ge_tensor_ptr); - } else { - MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; - ge_tensors.clear(); - return ge_tensors; - } - } - return ge_tensors; -} - -GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr &tensor, const std::string &format) { - // get tensor data type size - //获取变量tensors的data_type的size属性 - //检验tensor是否为空，并输出相应消息 - MS_EXCEPTION_IF_NULL(tensor); - //获取变量tensors的data_type的size属性 - size_t type_size = GetDataTypeSize(tensor->data_type()); - //判断其是否为空，若为空则输出错误日志"The Me Tensor data type size is wrong, type size is: "[*此处为type_size] - if (type_size == kErrorSize) { - MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; - return nullptr; - } - size_t elements_num = IntToSize(tensor->ElementsNum()); - - // get tensor buff size - //获取变量tensor的data_buff_size属性 - size_t data_buff_size = elements_num * type_size; - //检验其是否为空，若为空，则输出消息日志"The Me Tensor data buff size is 0." - if (data_buff_size == 0) { - MS_LOG(INFO) << "The Me Tensor data buff size is 0."; - } - // create ge tensor - //创建ge_tensor - //将tensors的属性写入到desc中 - auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); - //检验其是否为空，若为空，则输出错误日志 "Failed to get Tensor Desc" - //这一步主要检验获取data_type等情况 - if (desc == nullptr) { - MS_LOG(ERROR) << "Failed to get Tensor Desc"; - return nullptr; - } - //将desc和tensors有关属性存入tensors_ptr中 - GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); - //检验其是否为空，若不为空，则输出消息日志"Convert Me Tensor to Ge Tensor success!" - //此处为检验上述过程是否成功 - if (tensor_ptr != nullptr) { - MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; - } - return tensor_ptr; -} -//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector -std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors, - const std::vector &request_dims) { - //定义outputs - std::vector outputs; - //在index小于ge_tensors.size的范围内递增循环 - //定义me_tensor_ptr为空 - for (size_t index = 0; index < ge_tensors.size(); index++) { - MeTensorPtr me_tensor_ptr = nullptr; - //判断index和request_dims.size的大小 - //若后者较大，则将index对应位置的ge_tensors和request_dims存入me_tensor_ptr - //反之则定义empty_shape并将其与index对应位置的ge_tensors存入me_tensor_ptr - if (index < request_dims.size()) { - me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); - } else { - ShapeVector empty_shape; - me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); - } - //判断me_tensor_ptr是否为空 - //若不为空，则输出me_tensor_ptr相应内容 - //若为空，则出书错误日志"Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs - //此处为检验上述判断大小部分的功能是否成功实现 - if (me_tensor_ptr != nullptr) { - outputs.emplace_back(me_tensor_ptr); - } else { - MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; - return outputs; - } - } - return outputs; -} -//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector -std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors) { - //定义outputs - std::vector outputs; - //在index小于ge_tensors.size的范围内递增循环 - //定义me_tensor_ptr，储存经ConvertGeTensor处理后的index位ge_tensors - for (size_t index = 0; index < ge_tensors.size(); index++) { - MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); - //若me_tensor_ptr不为空，则进行相应output处理 - //否则，输出错误日志 "Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs - //主要检验ge_tensors与me_tensors的转化是否成功 - if (me_tensor_ptr != nullptr) { - outputs.emplace_back(me_tensor_ptr); - } else { - MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; - return outputs; - } - } - return outputs; -} -//在TransformUtil区域中创建名为:ConvertGeDataType的函数，返回值类型为自定义MeDataType -//此函数功能为检验type的类型，在type类型为DT_*的情况下返回对应的kNumberType* -//如果变量type不属于DT_*中的任何一种，则返回MeDataType::kTypeUnknown -MeDataType TransformUtil::ConvertGeDataType(const GeDataType &type) { - switch (type) { - case GeDataType::DT_FLOAT16: - return MeDataType::kNumberTypeFloat16; - case GeDataType::DT_FLOAT: - return MeDataType::kNumberTypeFloat32; - case GeDataType::DT_DOUBLE: - return MeDataType::kNumberTypeFloat64; - case GeDataType::DT_INT64: - return MeDataType::kNumberTypeInt64; - case GeDataType::DT_INT32: - return MeDataType::kNumberTypeInt32; - case GeDataType::DT_INT16: - return MeDataType::kNumberTypeInt16; - case GeDataType::DT_INT8: - return MeDataType::kNumberTypeInt8; - case GeDataType::DT_BOOL: - return MeDataType::kNumberTypeBool; - case GeDataType::DT_UINT8: - return MeDataType::kNumberTypeUInt8; - case GeDataType::DT_UINT16: - return MeDataType::kNumberTypeUInt16; - case GeDataType::DT_UINT32: - return MeDataType::kNumberTypeUInt32; - case GeDataType::DT_UINT64: - return MeDataType::kNumberTypeUInt64; - case GeDataType::DT_UNDEFINED: - case GeDataType::DT_DUAL_SUB_UINT8: - case GeDataType::DT_DUAL_SUB_INT8: - case GeDataType::DT_DUAL: - return MeDataType::kTypeUnknown; - default: - return MeDataType::kTypeUnknown; - } -} - -namespace { -//定义名为IsGeShapeCompatible的函数，返回值为bool -bool IsGeShapeCompatible(const GeShape &ge_shape, const ShapeVector &request_dims) { - //输出消息日志，说明当前Getensors的shape属性和我需要的shape属性 - MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); - MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); - - const int GE_DIMS = 4; - std::vector ge_dims = ge_shape.GetDims(); - //判断所需的size与当前size的大小，若前者较大，则输出错误日志 "Request shape's dims count greater than ge shape's" - //并返回false - if (request_dims.size() > ge_dims.size()) { - MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; - return false; - } - - // convert NHWC to NCHW 将NHWC转化为NCHW - //if括号中的内容为判断ge_tensors与re_quest的shape是否相容的条件 - //若符合条件则输出消息日志"Ge tensor shape and request shape is compatible"并返回true - if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && - (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { - MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; - return true; - } - //令i从0到request_dims.size范围内循环并判断ge_dims[i]与request_dims[i]是否相等 - //若不相等，则输出错误日志"Request shape's dims value not equal to ge shape's"并返回false - std::string::size_type i = 0; - for (; i < request_dims.size(); i++) { - if (ge_dims[i] != request_dims[i]) { - MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; - return false; - } - } - //判断i递增过程中对应的ge_dims[i]是否为1，并输出相应错误日志并返回false - for (; i < ge_dims.size(); i++) { - if (ge_dims[i] != 1) { - MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; - return false; - } - } - MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; - return true; -} -} // namespace - -//利用变量ge_dims实现变量me_dims向Geshape类型的转换 -GeShape TransformUtil::ConvertMeShape(const ShapeVector &me_dims) { - std::vector ge_dims; - (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); - return GeShape(ge_dims); -} -//利用变量me_dims实现变量ge_shape向ShapeVector类型的转换 -ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape) { - ShapeVector me_dims; - std::vector ge_dims = ge_shape.GetDims(); - (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); - return me_dims; -} -//在TransformUtil区域中创建名为ConvertGeShape的函数，返回值类型为自定义类型ShapeVector -ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape, const ShapeVector &request_dims) { - vector ret; - //判断ge_shape的对应属性是否为零，若为零则输出错误日志 "GeTensor's shape is scalar"并返回ret - if (ge_shape.GetDimNum() == 0) { - MS_LOG(DEBUG) << "GeTensor's shape is scalar"; - return ret; - } - //判断ge_shape是否可可兼容，若可兼容则用ret储存相应的值，否则输出相应错误日志并将ge_shape进行转换并用ret储存 - if (IsGeShapeCompatible(ge_shape, request_dims) == true) { - ret = request_dims; - } else { - MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; - ret = ConvertGeShape(ge_shape); - } - return ret; -} -//在TransformUtil区域中创建名为GenerateMeTensor的函数，返回值类型为自定义类型MeTensorPtr -MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const ShapeVector &me_dims, - const TypeId &me_type) { - MeTensor me_tensor(me_type, me_dims); - - // Get the writable data pointer of the tensor and cast it to its data type - //获取tensors的可写的数据指针并将其改写成其datatype - auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); - size_t me_data_size = static_cast(me_tensor.data().nbytes());//获取tensors的data size属性 - MS_EXCEPTION_IF_NULL(me_data_ptr);//检测me_data_ptr是否为空 - MS_EXCEPTION_IF_NULL(ge_tensor);//检测ge_tensor是否为空 - //判断me_data_size和ge_tensor的size的大小，若前者较小，则输出对应错误日志 - //"ME tensor data size["[*此处为me_data_size]" bytes] is less than GE tensor ["[*此处为ge_tensors的size]" bytes]" - //并返回nullptr - if (me_data_size < ge_tensor->GetSize()) { - MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" - << ge_tensor->GetSize() << " bytes]"; - return nullptr; - } - - // Copy or use the writable data pointer of the ME tensor - //复制或使用me_tensors的可写数据指针 - MS_EXCEPTION_IF_NULL(ge_tensor->GetData());//判断ge_tensors的data是否为空 - //判断ge_tensors的size属性是否为零，若为零，则输出相应错误日志并返回nullptr - if (ge_tensor->GetSize() == 0) { - MS_LOG(ERROR) << "GE tensor data size is zero!"; - return nullptr; - } - - // Use memcpy here, not memcpy_s, just because the size of ge_tensor may be bigger than 2GB - //因为ge_tensor的size可能大于2GB，所以在此处应用memcpy而不是memcpy_s - // which is the size limit of memcpy_s - //2GB为memcpy_s的极限容量 - (void)memcpy(me_data_ptr, ge_tensor->GetData(), ge_tensor->GetSize()); - - return make_shared(me_tensor); -} -//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr -MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr &ge_tensor) { - MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 - GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape - vector me_dims = ConvertGeShape(ge_shape); - - TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype - //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr - if (type_id == MeDataType::kTypeUnknown) { - MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " - << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - return nullptr; - } - return GenerateMeTensor(ge_tensor, me_dims, type_id); -} - -// if request_dims is empty, use ge tensor's shape, otherwise convert to request shape -//若 request_dim为空，则使用ge_tensors的shape，用其他的方式转化成要求的shape -//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr -MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const ShapeVector &request_dims) { - MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 - GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape - vector me_dims = ConvertGeShape(ge_shape, request_dims); - //输出信息日志 "GE tensor type is [*此处为ge_tensors的datatype]" - MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - // Create a tensor with wanted data type and shape - TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype - //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr - if (type_id == MeDataType::kTypeUnknown) { - MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " - << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - return nullptr; - } - return GenerateMeTensor(ge_tensor, me_dims, type_id); -} -//在TransformUtil区域中创建名为PrintGeTensor的函数，返回值类型为std::string -std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { - std::string ret; - //检验变量ge_tensors是否为空，若为空则输出错误日志"Input ge tensor is nullptr"并返回ret - if (ge_tensor == nullptr) { - MS_LOG(ERROR) << "Input ge tensor is nullptr"; - return ret; - } - //ge_tensors不为空执行以下语句 - //输出信息日志"Ge Tensor data type is : [*此处为ge_tensors的data_type]" - //对于ge_trensors的data_type进行比对，若其符合GeDataType::DT_*等一系列的值，则令ret等于对应的值并结束语句 - //若其不符合这一系列的值，则输出错误日志"Unsupported to print type:"[*此处为ge_tensors的data type]" ge tensor";并结束语句 - MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - switch (static_cast(ge_tensor->GetTensorDesc().GetDataType())) { - case GeDataType::DT_UINT32: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_FLOAT: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_INT32: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_DOUBLE: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_INT64: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_UINT64: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_INT16: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_UINT16: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_DUAL_SUB_INT8: - case GeDataType::DT_INT8: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_UINT8: - case GeDataType::DT_DUAL_SUB_UINT8: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_FLOAT16: - case GeDataType::DT_BOOL: - case GeDataType::DT_UNDEFINED: - case GeDataType::DT_DUAL: - default: - MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) - << " ge tensor"; - break; - } - return ret; -} -} // namespace transform -} // namespace mindspore -- 2.34.1 From 160079df520d8e740cf45db208e96eb99fbb7b70 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:45:14 +0800 Subject: [PATCH 076/160] ADD file via upload --- .../ccsrc/minddata/mindrecord/include/else | 549 ++++++++++++++++++ 1 file changed, 549 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/else diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/else b/mindspore2022/ccsrc/minddata/mindrecord/include/else new file mode 100644 index 00000000000..eda89bd3759 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/else @@ -0,0 +1,549 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/util.h" + +#include +#include + +#include "securec/include/securec.h" +#include "include/common/utils/convert_utils.h" +#include "include/common/utils/utils.h" + +namespace mindspore { +namespace transform { +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; + +const size_t kErrorSize = 0; +//在TransformUtil区域中创建名为ConvertIntToList的函数，返回值类型为vector，用于检测size的长度。 +//若长度小于等于零则弹出MS_LOG警告日志 "size <= 0"并直接返回list +//若长度大于零则用list对data中的内容进行push_back操作 +vector TransformUtil::ConvertIntToList(int64_t data, int size) { + vector list{}; + if (size <= 0) { + MS_LOG(WARNING) << "size <= 0"; + return list; + } + for (int i = 0; i < size; ++i) { + list.push_back(data); + } + return list; +} + +static std::map datatype_trans_map = { + {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, + {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, + {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, + {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, + {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, + {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; +//在TransformUtil区域中创建名为ConvertDataType的函数，返回值类型为自定义GeDataType +//输出MS_LOG调试日志"Convert me data type: "[*此处为对type进行TypeIdLabel(type)的结果]ge data type" +//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 +//若不一致，则返回datatype_trans_map[type] +//若一致则返回DT_UNDEFINED +GeDataType TransformUtil::ConvertDataType(const MeDataType &type) { + MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; + if (datatype_trans_map.find(type) != datatype_trans_map.end()) { + return datatype_trans_map[type]; + } else { + return GeDataType::DT_UNDEFINED; + } +} + +static std::map datatype_size_map = { + {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float + {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, + {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, + {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, + {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, + {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; +//在TransformUtil区域中创建名为GetDataTypeSize的函数，返回值类型为自定义size_t +//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 +//若不一致，则返回datatype_trans_map[type] +//若一致，则输出错误日志 "Illegal tensor data type!"并返回kErrorSize +size_t TransformUtil::GetDataTypeSize(const MeDataType &type) { + if (datatype_size_map.find(type) != datatype_size_map.end()) { + return datatype_size_map[type]; + } else { + MS_LOG(ERROR) << "Illegal tensor data type!"; + return kErrorSize; + } +} +//在TransformUtil区域中创建名为:ConvertFormat的函数，返回值类型为自定义GeFormat +//判断format与 kOpFormat_NCHW等kOpFormat_*是否相等，若相等，则返回 kOpFormat_NCHW等kOpFormat_* +//若与一系列kOpFormat_*都不相等，则输出错误日志 "Illegal tensor data format: ("[*此处为format变量]"). Use ND format instead." +//并返回FORMAT_ND +GeFormat TransformUtil::ConvertFormat(const string &format) { + if (format == kOpFormat_NCHW) { + return GeFormat::FORMAT_NCHW; + } else if (format == kOpFormat_NDHWC) { + return GeFormat::FORMAT_NDHWC; + } else if (format == kOpFormat_NCDHW) { + return GeFormat::FORMAT_NCDHW; + } else if (format == kOpFormat_DHWNC) { + return GeFormat::FORMAT_DHWNC; + } else if (format == kOpFormat_DHWCN) { + return GeFormat::FORMAT_DHWCN; + } else if (format == kOpFormat_NC1HWC0) { + return GeFormat::FORMAT_NC1HWC0; + } else if (format == kOpFormat_NHWC) { + return GeFormat::FORMAT_NHWC; + } else if (format == kOpFormat_HWCN) { + return GeFormat::FORMAT_HWCN; + } else if (format == kOpFormat_ND) { + return GeFormat::FORMAT_ND; + } else { + MS_LOG(ERROR) << "Illegal tensor data format: (" << format << "). Use ND format instead."; + return GeFormat::FORMAT_ND; + } +} +//创建名为IntegerCastFunc的函数，返回值类型为自定义static int64_t +//返回执行static_cast()操作后的size_t的temp +static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } + +//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr TransformUtil::GetGeTensorDesc(const ShapeVector &me_shape, const MeDataType &me_type, + const std::string &format) { + // convert me shape to ge shape + //将me_shape转化为ge_shape + //定义ge_shape + std::vector ge_shape; + //若me_shape.size()=1，则对其第[0]位进行push_back操作 + //否则，则根据me_shape的size属性设置ge_shape的size属性 + if (me_shape.size() == 1) { + ge_shape.push_back(static_cast(me_shape[0])); + } else { + ge_shape.resize(me_shape.size()); + (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); + } + //若shape.GetDimNum()=0，则输出信息日志"The dims size of Ge tensor is zero" + GeShape shape(ge_shape); + if (shape.GetDimNum() == 0) { + MS_LOG(INFO) << "The dims size of Ge tensor is zero"; + } + // convert me format to ge format + //将me_format转化为ge_format + //若ge_format符合相应条件则输出信息日志 "Set ND data format" + GeFormat ge_format = ConvertFormat(format); + if (ge_format == GeFormat::FORMAT_ND) { + MS_LOG(INFO) << "Set ND data format"; + } + // convert me datatype to ge datatype + //将me_datatype转化为ge_datatype + //若data_type符合相应条件则输出错误日志"undefined data type :"[*此处为me_type] + //并返回 nullptr + GeDataType data_type = ConvertDataType(me_type); + if (data_type == GeDataType::DT_UNDEFINED) { + MS_LOG(ERROR) << "undefined data type :" << me_type; + return nullptr; + } + //进行make_shared操作，若desc变量为空则输出错误日志 "Create GeTensorDesc failed!" + //并返回 nullptr + //若不为空则输出信息日志"SetRealDimCnt is :"[*此处为 me_shape.size()] + //并对desc进行SetRealDimCnt操作 + //并返回desc + //这一步在检验转换是否成功并且转换后对象的size属性正确 + auto desc = std::make_shared(shape, ge_format, data_type); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create GeTensorDesc failed!"; + return nullptr; + } + MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); + desc->SetRealDimCnt(SizeToInt(me_shape.size())); + return desc; +} + +// if failed, return empty vector. +//如果失败了，则返回空的vector +//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr +std::vector TransformUtil::ConvertInputTensors(const std::vector &me_tensors, + const std::string &format) { + //定义ge_tensors + std::vector ge_tensors; + //在index小于me_tensors.size的范围内递增循环 + //输出消息日志 "me_tensor "[*此处为index(tensors的位数)]" 's data size is: "[*此处为index对应位的tensors的size] + for (size_t index = 0; index < me_tensors.size(); index++) { + MS_EXCEPTION_IF_NULL(me_tensors[index]); + MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); + //定义变量shape为index对应位tensors的shape + //将shape一个个写入shape_str中 + auto shape = me_tensors[index]->shape(); + std::string shape_str; + for (size_t i = 0; i < shape.size(); i++) { + shape_str += std::to_string(shape[i]); + shape_str += " "; + } + //输出消息日志 "me_tensor "[*此处为index(位数)]" 's shape is:"[*此处为index对应位的tensors的shape] + //输出消息日志"me_tensor "[*此处为index(位数)]" 's type is: "[*此处为index对应位的tensors的type] + MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; + MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); + //定义ge_tensor_ptr + auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); + //如果ge_tensor_ptr为空，则执行对应操作 + //否则，输出错误日志 "Convert me_tensor "[*此处为index(位数)]" to Ge Tensor failed!" + //清空ge_tensors并返回 + //这一步为检验此前过程是否成功 + if (ge_tensor_ptr != nullptr) { + ge_tensors.emplace_back(ge_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; + ge_tensors.clear(); + return ge_tensors; + } + } + return ge_tensors; +} + +GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr &tensor, const std::string &format) { + // get tensor data type size + //获取变量tensors的data_type的size属性 + //检验tensor是否为空，并输出相应消息 + MS_EXCEPTION_IF_NULL(tensor); + //获取变量tensors的data_type的size属性 + size_t type_size = GetDataTypeSize(tensor->data_type()); + //判断其是否为空，若为空则输出错误日志"The Me Tensor data type size is wrong, type size is: "[*此处为type_size] + if (type_size == kErrorSize) { + MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; + return nullptr; + } + size_t elements_num = IntToSize(tensor->ElementsNum()); + + // get tensor buff size + //获取变量tensor的data_buff_size属性 + size_t data_buff_size = elements_num * type_size; + //检验其是否为空，若为空，则输出消息日志"The Me Tensor data buff size is 0." + if (data_buff_size == 0) { + MS_LOG(INFO) << "The Me Tensor data buff size is 0."; + } + // create ge tensor + //创建ge_tensor + //将tensors的属性写入到desc中 + auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + //检验其是否为空，若为空，则输出错误日志 "Failed to get Tensor Desc" + //这一步主要检验获取data_type等情况 + if (desc == nullptr) { + MS_LOG(ERROR) << "Failed to get Tensor Desc"; + return nullptr; + } + //将desc和tensors有关属性存入tensors_ptr中 + GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); + //检验其是否为空，若不为空，则输出消息日志"Convert Me Tensor to Ge Tensor success!" + //此处为检验上述过程是否成功 + if (tensor_ptr != nullptr) { + MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; + } + return tensor_ptr; +} +//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors, + const std::vector &request_dims) { + //定义outputs + std::vector outputs; + //在index小于ge_tensors.size的范围内递增循环 + //定义me_tensor_ptr为空 + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = nullptr; + //判断index和request_dims.size的大小 + //若后者较大，则将index对应位置的ge_tensors和request_dims存入me_tensor_ptr + //反之则定义empty_shape并将其与index对应位置的ge_tensors存入me_tensor_ptr + if (index < request_dims.size()) { + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); + } else { + ShapeVector empty_shape; + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); + } + //判断me_tensor_ptr是否为空 + //若不为空，则输出me_tensor_ptr相应内容 + //若为空，则出书错误日志"Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs + //此处为检验上述判断大小部分的功能是否成功实现 + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} +//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors) { + //定义outputs + std::vector outputs; + //在index小于ge_tensors.size的范围内递增循环 + //定义me_tensor_ptr，储存经ConvertGeTensor处理后的index位ge_tensors + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); + //若me_tensor_ptr不为空，则进行相应output处理 + //否则，输出错误日志 "Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs + //主要检验ge_tensors与me_tensors的转化是否成功 + if (me_tensor_ptr != nullptr) { + outputs.emplace_back(me_tensor_ptr); + } else { + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; +} +//在TransformUtil区域中创建名为:ConvertGeDataType的函数，返回值类型为自定义MeDataType +//此函数功能为检验type的类型，在type类型为DT_*的情况下返回对应的kNumberType* +//如果变量type不属于DT_*中的任何一种，则返回MeDataType::kTypeUnknown +MeDataType TransformUtil::ConvertGeDataType(const GeDataType &type) { + switch (type) { + case GeDataType::DT_FLOAT16: + return MeDataType::kNumberTypeFloat16; + case GeDataType::DT_FLOAT: + return MeDataType::kNumberTypeFloat32; + case GeDataType::DT_DOUBLE: + return MeDataType::kNumberTypeFloat64; + case GeDataType::DT_INT64: + return MeDataType::kNumberTypeInt64; + case GeDataType::DT_INT32: + return MeDataType::kNumberTypeInt32; + case GeDataType::DT_INT16: + return MeDataType::kNumberTypeInt16; + case GeDataType::DT_INT8: + return MeDataType::kNumberTypeInt8; + case GeDataType::DT_BOOL: + return MeDataType::kNumberTypeBool; + case GeDataType::DT_UINT8: + return MeDataType::kNumberTypeUInt8; + case GeDataType::DT_UINT16: + return MeDataType::kNumberTypeUInt16; + case GeDataType::DT_UINT32: + return MeDataType::kNumberTypeUInt32; + case GeDataType::DT_UINT64: + return MeDataType::kNumberTypeUInt64; + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL_SUB_UINT8: + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_DUAL: + return MeDataType::kTypeUnknown; + default: + return MeDataType::kTypeUnknown; + } +} + +namespace { +//定义名为IsGeShapeCompatible的函数，返回值为bool +bool IsGeShapeCompatible(const GeShape &ge_shape, const ShapeVector &request_dims) { + //输出消息日志，说明当前Getensors的shape属性和我需要的shape属性 + MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); + MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); + + const int GE_DIMS = 4; + std::vector ge_dims = ge_shape.GetDims(); + //判断所需的size与当前size的大小，若前者较大，则输出错误日志 "Request shape's dims count greater than ge shape's" + //并返回false + if (request_dims.size() > ge_dims.size()) { + MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; + return false; + } + + // convert NHWC to NCHW 将NHWC转化为NCHW + //if括号中的内容为判断ge_tensors与re_quest的shape是否相容的条件 + //若符合条件则输出消息日志"Ge tensor shape and request shape is compatible"并返回true + if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && + (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; + } + //令i从0到request_dims.size范围内循环并判断ge_dims[i]与request_dims[i]是否相等 + //若不相等，则输出错误日志"Request shape's dims value not equal to ge shape's"并返回false + std::string::size_type i = 0; + for (; i < request_dims.size(); i++) { + if (ge_dims[i] != request_dims[i]) { + MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; + return false; + } + } + //判断i递增过程中对应的ge_dims[i]是否为1，并输出相应错误日志并返回false + for (; i < ge_dims.size(); i++) { + if (ge_dims[i] != 1) { + MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; + return false; + } + } + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; +} +} // namespace + +//利用变量ge_dims实现变量me_dims向Geshape类型的转换 +GeShape TransformUtil::ConvertMeShape(const ShapeVector &me_dims) { + std::vector ge_dims; + (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); + return GeShape(ge_dims); +} +//利用变量me_dims实现变量ge_shape向ShapeVector类型的转换 +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape) { + ShapeVector me_dims; + std::vector ge_dims = ge_shape.GetDims(); + (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); + return me_dims; +} +//在TransformUtil区域中创建名为ConvertGeShape的函数，返回值类型为自定义类型ShapeVector +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape, const ShapeVector &request_dims) { + vector ret; + //判断ge_shape的对应属性是否为零，若为零则输出错误日志 "GeTensor's shape is scalar"并返回ret + if (ge_shape.GetDimNum() == 0) { + MS_LOG(DEBUG) << "GeTensor's shape is scalar"; + return ret; + } + //判断ge_shape是否可可兼容，若可兼容则用ret储存相应的值，否则输出相应错误日志并将ge_shape进行转换并用ret储存 + if (IsGeShapeCompatible(ge_shape, request_dims) == true) { + ret = request_dims; + } else { + MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; + ret = ConvertGeShape(ge_shape); + } + return ret; +} +//在TransformUtil区域中创建名为GenerateMeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const ShapeVector &me_dims, + const TypeId &me_type) { + MeTensor me_tensor(me_type, me_dims); + + // Get the writable data pointer of the tensor and cast it to its data type + //获取tensors的可写的数据指针并将其改写成其datatype + auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); + size_t me_data_size = static_cast(me_tensor.data().nbytes());//获取tensors的data size属性 + MS_EXCEPTION_IF_NULL(me_data_ptr);//检测me_data_ptr是否为空 + MS_EXCEPTION_IF_NULL(ge_tensor);//检测ge_tensor是否为空 + //判断me_data_size和ge_tensor的size的大小，若前者较小，则输出对应错误日志 + //"ME tensor data size["[*此处为me_data_size]" bytes] is less than GE tensor ["[*此处为ge_tensors的size]" bytes]" + //并返回nullptr + if (me_data_size < ge_tensor->GetSize()) { + MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" + << ge_tensor->GetSize() << " bytes]"; + return nullptr; + } + + // Copy or use the writable data pointer of the ME tensor + //复制或使用me_tensors的可写数据指针 + MS_EXCEPTION_IF_NULL(ge_tensor->GetData());//判断ge_tensors的data是否为空 + //判断ge_tensors的size属性是否为零，若为零，则输出相应错误日志并返回nullptr + if (ge_tensor->GetSize() == 0) { + MS_LOG(ERROR) << "GE tensor data size is zero!"; + return nullptr; + } + + // Use memcpy here, not memcpy_s, just because the size of ge_tensor may be bigger than 2GB + //因为ge_tensor的size可能大于2GB，所以在此处应用memcpy而不是memcpy_s + // which is the size limit of memcpy_s + //2GB为memcpy_s的极限容量 + (void)memcpy(me_data_ptr, ge_tensor->GetData(), ge_tensor->GetSize()); + + return make_shared(me_tensor); +} +//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr &ge_tensor) { + MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape + vector me_dims = ConvertGeShape(ge_shape); + + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype + //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} + +// if request_dims is empty, use ge tensor's shape, otherwise convert to request shape +//若 request_dim为空，则使用ge_tensors的shape，用其他的方式转化成要求的shape +//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const ShapeVector &request_dims) { + MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape + vector me_dims = ConvertGeShape(ge_shape, request_dims); + //输出信息日志 "GE tensor type is [*此处为ge_tensors的datatype]" + MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + // Create a tensor with wanted data type and shape + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype + //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} +//在TransformUtil区域中创建名为PrintGeTensor的函数，返回值类型为std::string +std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { + std::string ret; + //检验变量ge_tensors是否为空，若为空则输出错误日志"Input ge tensor is nullptr"并返回ret + if (ge_tensor == nullptr) { + MS_LOG(ERROR) << "Input ge tensor is nullptr"; + return ret; + } + //ge_tensors不为空执行以下语句 + //输出信息日志"Ge Tensor data type is : [*此处为ge_tensors的data_type]" + //对于ge_trensors的data_type进行比对，若其符合GeDataType::DT_*等一系列的值，则令ret等于对应的值并结束语句 + //若其不符合这一系列的值，则输出错误日志"Unsupported to print type:"[*此处为ge_tensors的data type]" ge tensor";并结束语句 + MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + switch (static_cast(ge_tensor->GetTensorDesc().GetDataType())) { + case GeDataType::DT_UINT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DOUBLE: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_INT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT8: + case GeDataType::DT_DUAL_SUB_UINT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT16: + case GeDataType::DT_BOOL: + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL: + default: + MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) + << " ge tensor"; + break; + } + return ret; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 5a3a19aeaa936cbcc46184fa12310f4f9fe5d693 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:45:33 +0800 Subject: [PATCH 077/160] ADD file via upload --- .../minddata/mindrecord/include/shard_utils.h | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h new file mode 100644 index 00000000000..056d086da08 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h @@ -0,0 +1,210 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ + +#include +#include +#include +#include +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_error.h" +#include "nlohmann/json.hpp" +#include "./sqlite3.h" +#include "utils/log_adapter.h" + +/* To be used when dlog is ok #include "./slog.h" */ +#ifdef DEBUG +#define MS_ASSERT(f) assert(f) +#else +#define MS_ASSERT(f) ((void)0) +#endif + +namespace mindspore {// +namespace mindrecord { +using json = nlohmann::json; + +//定义int型常量KInt1，KInt2，KInt3，kUnsignedInt4 +const int kInt0 = 0; +const int kInt1 = 1; +const int kInt2 = 2; +const int kInt3 = 3; +const int kUnsignedInt4 = 4; + +enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel }; + +const char kVersion[] = "3.0"; //将3.0赋给字符型数组kVersion[] +const std::vector kSupportedVersion = {"2.0", kVersion}; //将2.0，kVersion赋给vector kSupportedVersion + +enum ShardType { + kNLP = 0, + kCV = 1, +}; + +enum TaskType { + kCommonTask = 0, + kPaddedTask = 1, +}; +enum SamplerType { kCustomTopNSampler, kCustomTopPercentSampler, kSubsetRandomSampler, kPKSampler, kSubsetSampler }; + +enum ShuffleType { kShuffleCategory, kShuffleSample }; + +const double kEpsilon = 1e-7; + +const int kThreadNumber = 14; + +// Shard default parameters Shard默认参数 +const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB +const uint64_t kDefaultPageSize = 1 << 25; // 32MB + +// HeaderSize [16KB, 128MB] +const int kMinHeaderSize = 1 << 14; // 16KB +const int kMaxHeaderSize = 1 << 27; // 128MB + +// PageSize [32KB, 256MB] +const int kMinPageSize = 1 << 15; // 32KB +const int kMaxPageSize = 1 << 28; // 256MB + +// used by value length / schema id length / statistic id length ... 由值长度/架构id长度/统计id长度使用。。。 +const uint64_t kInt64Len = 8; + +// Minimum file size 最小文件大小 +const uint64_t kMinFileSize = kInt64Len; + +const int kMinShardCount = 1; +const int kMaxShardCount = 1000; // write +const int kMaxFileCount = 4096; // read + +const int kMinConsumerCount = 1; +const int kMaxConsumerCount = 128; + +const int kMaxSchemaCount = 1; +const int kMaxThreadCount = 32; +const int kMaxFieldCount = 100; + +// Minimum free disk size 最小可用磁盘大小 +const int kMinFreeDiskSize = 10; // 10M + +// dummy json +const json kDummyId = R"({"id": 0})"_json; + +// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) 将模式中的类型转换为sqlite3中的类型（NULL、INTEGER、REAL、TEXT、BLOB） +const std::unordered_map kDbJsonMap = { + {"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"}, + {"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"}, + {"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"}, + {"bytes", "BLOB"}}; + +const char kPoint = '.'; + +const char kPathSeparator = +#if defined(_WIN32) || defined(_WIN64) + '\\'; +#else + '/'; +#endif + +// field type used by check schema validation 检查架构验证使用的字段类型 +const std::set kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"}; + +// can be searched field list 可搜索字段列表 +const std::set kScalarFieldTypeSet = {"string", "int32", "int64", "float32", "float64"}; + +// number field list 数字字段列表 +const std::set kNumberFieldTypeSet = {"int32", "int64", "float32", "float64"}; + +const std::unordered_map kTypesMap = { + {"bool", "int32"}, {"int8", "int32"}, {"uint8", "bytes"}, {"int16", "int32"}, + {"uint16", "int32"}, {"int32", "int32"}, {"uint32", "int64"}, {"int64", "int64"}, + {"float16", "float32"}, {"float32", "float32"}, {"float64", "float64"}, {"string", "string"}}; + +/// \brief the max number of samples to enable lazy load 启用延迟加载的最大样本数 +const uint32_t LAZY_LOAD_THRESHOLD = 5000000; + +/// \brief split a string using a character 使用字符拆分字符串 +/// \param[in] field target string 目标字符串 +/// \param[in] separator a character for splitting 用于拆分的字符 +/// \return vector type result 矢量类型结果 +std::vector StringSplit(const std::string &field, char separator); + +/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' 验证字段名由“0-9”或“a-z”或“a-z”或“_”或“-”组成 +/// \param[in] str target string 目标字符串 +/// \return +bool ValidateFieldName(const std::string &str); + +/// \brief get the filename by the path 通过路径获取文件名 +/// \param s file path 文件路径 +/// \param fn_ptr shared ptr of file name 文件名的共享指针 +/// \return Status +Status GetFileName(const std::string &path, std::shared_ptr *fn_ptr); + +/// \brief get parent dir 获取父目录 +/// \param path file path 文件路径 +/// \param pd_ptr shared ptr of parent path 父路径的共享指针 +/// \return Status +Status GetParentDir(const std::string &path, std::shared_ptr *pd_ptr); + +bool CheckIsValidUtf8(const std::string &str); + +/// \brief judge if a path is legal file 判断路径是否为合法文件 +/// \param path file path 文件路径 +/// \return Whether the path is legal or not 路径是否合法 +bool IsLegalFile(const std::string &path); + +enum DiskSizeType { kTotalSize = 0, kFreeSize }; + +/// \brief get the free space about the disk 获取磁盘的可用空间 +/// \param str_dir file path 文件路径 +/// \param disk_type: kTotalSize / kFreeSize +/// \param size: shared ptr of size in Megabytes 以MB为单位的共享指针 +/// \return Status +Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr *size); + +/// \brief get the max hardware concurrency //获取最大硬件并发 +/// \return max concurrency 最大并发数 +uint32_t GetMaxThreadNum(); + +/// \brief get absolute path of all mindrecord files 获取所有mindrecord文件的绝对路径 +/// \param path path to one fo mindrecord files 一个fo-mindrecord文件的路径 +/// \param addresses relative path of all mindrecord files 所有思维记录文件的相对路径 +/// \param ds shared ptr of vector of absolute path 绝对路径矢量的共享指针 +/// \return Status +Status GetDatasetFiles(const std::string &path, const json &addresses, std::shared_ptr> *ds); +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -- 2.34.1 From d5f792ba34b2b97926f085717f2d6e67acf19b38 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Fri, 14 Oct 2022 21:45:50 +0800 Subject: [PATCH 078/160] ADD file via upload --- .../mindrecord/include/shard_category.h | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_category.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_category.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_category.h new file mode 100644 index 00000000000..876833a5bec --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_category.h @@ -0,0 +1,63 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_CATEGORY_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_CATEGORY_H_ + +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_operator.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardCategory : public ShardOperator { + public: + explicit ShardCategory(const std::vector> &categories, + int64_t num_elements = std::numeric_limits::max(), bool replacement = false); //定义64位int型num_elements，定义布尔型replacement为false + + ShardCategory(const std::string &category_field, int64_t num_elements, + int64_t num_categories = std::numeric_limits::max(), bool replacement = false); + + ~ShardCategory() override{}; + + const std::vector> &GetCategories() const { return categories_; } //返回categories的函数 + + const std::string GetCategoryField() const { return category_field_; } //返回category_field_的函数 + + int64_t GetNumElements() const { return num_elements_; } //返回num_elements_的函数 + + int64_t GetNumCategories() const { return num_categories_; } //返回num_categories_的函数 + + bool GetReplacement() const { return replacement_; } //返回replacement_的函数 + + Status Execute(ShardTaskList &tasks) override; + + int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override; + + private: //私有 + std::vector> categories_; + std::string category_field_; + int64_t num_elements_; + int64_t num_categories_; + bool replacement_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_CATEGORY_H_ -- 2.34.1 From 6d9b87667d3256383274c0ed8155fcc8a4ea4f4c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:39:14 +0800 Subject: [PATCH 079/160] ADD file via upload --- .../minddata/mindrecord/include/meta/metaing | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing new file mode 100644 index 00000000000..462023d09b6 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing @@ -0,0 +1,54 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_category.h" + +namespace mindspore { +namespace mindrecord { +ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, + bool replacement) + : categories_(categories), + category_field_(""), + num_elements_(num_elements), + num_categories_(0), + replacement_(replacement) {} + +ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, + bool replacement) + : categories_({}), + category_field_(category_field), + num_elements_(num_elements), + num_categories_(num_categories), + replacement_(replacement) {} + +Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); } + +int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (dataset_size == 0) return dataset_size; + if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { + num_classes = std::min(num_categories_, num_classes); + if (num_classes == 0) { + return 0; + } + if (num_elements_ > std::numeric_limits::max() / num_classes) { + return -1; + } + return num_classes * num_elements_; + } + return 0; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 95c8d88cce80422099f3c91a02d001f8d04e7c9a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:39:38 +0800 Subject: [PATCH 080/160] ADD file via upload --- .../mindrecord/include/meta/shard_category.cc | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc new file mode 100644 index 00000000000..462023d09b6 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_category.h" + +namespace mindspore { +namespace mindrecord { +ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, + bool replacement) + : categories_(categories), + category_field_(""), + num_elements_(num_elements), + num_categories_(0), + replacement_(replacement) {} + +ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, + bool replacement) + : categories_({}), + category_field_(category_field), + num_elements_(num_elements), + num_categories_(num_categories), + replacement_(replacement) {} + +Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); } + +int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (dataset_size == 0) return dataset_size; + if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { + num_classes = std::min(num_categories_, num_classes); + if (num_classes == 0) { + return 0; + } + if (num_elements_ > std::numeric_limits::max() / num_classes) { + return -1; + } + return num_classes * num_elements_; + } + return 0; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From c064a151314c63e78a6c64068f7dfd9b9bd18bf4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:40:02 +0800 Subject: [PATCH 081/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing' --- .../minddata/mindrecord/include/meta/metaing | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing deleted file mode 100644 index 462023d09b6..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/metaing +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_category.h" - -namespace mindspore { -namespace mindrecord { -ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, - bool replacement) - : categories_(categories), - category_field_(""), - num_elements_(num_elements), - num_categories_(0), - replacement_(replacement) {} - -ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, - bool replacement) - : categories_({}), - category_field_(category_field), - num_elements_(num_elements), - num_categories_(num_categories), - replacement_(replacement) {} - -Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); } - -int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (dataset_size == 0) return dataset_size; - if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { - num_classes = std::min(num_categories_, num_classes); - if (num_classes == 0) { - return 0; - } - if (num_elements_ > std::numeric_limits::max() / num_classes) { - return -1; - } - return num_classes * num_elements_; - } - return 0; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 07d71c3e27181a13dfce36fbe846d527aa2a3205 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:40:34 +0800 Subject: [PATCH 082/160] ADD file via upload --- .../include/meta/shard_distributed_sample.cc | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc new file mode 100644 index 00000000000..cb1d06efdb2 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc @@ -0,0 +1,80 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_distributed_sample.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, int64_t no_of_padded_samples, bool shuffle, + uint32_t seed, int64_t no_of_samples, int64_t offset) + : ShardSample(1, num_shards, shard_id, no_of_samples, offset), + shuffle_(shuffle), + no_of_padded_samples_(no_of_padded_samples), + first_epoch_(true) { + shuffle_op_ = std::make_shared(seed, kShuffleSample); +} + +ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, bool shuffle, uint32_t seed, + int64_t no_of_samples, int64_t offset) + : ShardDistributedSample(num_shards, shard_id, 0, shuffle, seed, no_of_samples, offset) {} + +int64_t ShardDistributedSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (no_of_padded_samples_ <= 0) { + int64_t res = 0; + if (dataset_size % denominator_ == 0) { + res = dataset_size / denominator_ * numerator_; + } else { + res = dataset_size / denominator_ * numerator_ + 1; + } + return no_of_samples_ == 0 ? res : std::min(no_of_samples_, res); + } else { + auto padded_size = dataset_size + no_of_padded_samples_; + if (padded_size % denominator_ == 0) { + return padded_size / denominator_ * numerator_; + } else { + return -1; + } + } + return 0; +} + +Status ShardDistributedSample::PreExecute(ShardTaskList &tasks) { + auto total_no = tasks.Size(); + if (no_of_padded_samples_ > 0 && first_epoch_) { + CHECK_FAIL_RETURN_UNEXPECTED(total_no % denominator_ == 0, + "Invalid data, the size of dataset and padded samples: " + std::to_string(total_no) + + " can not be divisible by the value of 'num_shards': " + + std::to_string(denominator_) + ".\n Please adjust the value of 'num_padded'."); + } + if (first_epoch_) { + first_epoch_ = false; + task_ = tasks; + } else { + tasks = task_; + } + if (shuffle_ == true) { + shuffle_op_->SetShardSampleCount(GetShardSampleCount()); + shuffle_op_->UpdateShuffleMode(GetShuffleMode()); + RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 7ef11478c4b52a312877ca2aca468b33a03061b2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:41:03 +0800 Subject: [PATCH 083/160] ADD file via upload --- .../mindrecord/include/meta/shard_column.cc | 500 ++++++++++++++++++ 1 file changed, 500 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc new file mode 100644 index 00000000000..9626acd9404 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc @@ -0,0 +1,500 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_column.h" + +#include "utils/ms_utils.h" +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_error.h" + +namespace mindspore { +namespace mindrecord { +ShardColumn::ShardColumn(const std::shared_ptr &shard_header, bool compress_integer) { + auto first_schema = shard_header->GetSchemas()[0]; + json schema_json = first_schema->GetSchema(); + Init(schema_json, compress_integer); +} + +ShardColumn::ShardColumn(const json &schema_json, bool compress_integer) { Init(schema_json, compress_integer); } + +void ShardColumn::Init(const json &schema_json, bool compress_integer) { + auto schema = schema_json["schema"]; + auto blob_fields = schema_json["blob_fields"]; + + bool has_integer_array = false; + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + const std::string &column_name = it.key(); + column_name_.push_back(column_name); + + json it_value = it.value(); + + std::string str_type = it_value["type"]; + column_data_type_.push_back(ColumnDataTypeMap.at(str_type)); + if (it_value.find("shape") != it_value.end()) { + std::vector vec(it_value["shape"].size()); + std::copy(it_value["shape"].begin(), it_value["shape"].end(), vec.begin()); + column_shape_.push_back(vec); + if (str_type == "int32" || str_type == "int64") { + has_integer_array = true; + } + } else { + std::vector vec = {}; + column_shape_.push_back(vec); + } + } + + for (uint64_t i = 0; i < column_name_.size(); i++) { + column_name_id_[column_name_[i]] = i; + } + + for (const auto &field : blob_fields) { + blob_column_.push_back(field); + } + + for (uint64_t i = 0; i < blob_column_.size(); i++) { + blob_column_id_[blob_column_[i]] = i; + } + + has_compress_blob_ = (compress_integer && has_integer_array); + num_blob_column_ = blob_column_.size(); +} + +Status ShardColumn::GetColumnTypeByName(const std::string &column_name, ColumnDataType *column_data_type, + uint64_t *column_data_type_size, std::vector *column_shape, + ColumnCategory *column_category) { + RETURN_UNEXPECTED_IF_NULL(column_data_type); + RETURN_UNEXPECTED_IF_NULL(column_data_type_size); + RETURN_UNEXPECTED_IF_NULL(column_shape); + RETURN_UNEXPECTED_IF_NULL(column_category); + // Skip if column not found + *column_category = CheckColumnName(column_name); + CHECK_FAIL_RETURN_UNEXPECTED(*column_category != ColumnNotFound, + "[Internal ERROR] the type of column: " + column_name + " can not found."); + + // Get data type and size + auto column_id = column_name_id_[column_name]; + *column_data_type = column_data_type_[column_id]; + *column_data_type_size = ColumnDataTypeSize[*column_data_type]; + *column_shape = column_shape_[column_id]; + return Status::OK(); +} + +Status ShardColumn::GetColumnValueByName(const std::string &column_name, const std::vector &columns_blob, + const json &columns_json, const unsigned char **data, + std::unique_ptr *data_ptr, uint64_t *const n_bytes, + ColumnDataType *column_data_type, uint64_t *column_data_type_size, + std::vector *column_shape) { + RETURN_UNEXPECTED_IF_NULL(column_data_type); + RETURN_UNEXPECTED_IF_NULL(column_data_type_size); + RETURN_UNEXPECTED_IF_NULL(column_shape); + // Skip if column not found + auto column_category = CheckColumnName(column_name); + CHECK_FAIL_RETURN_UNEXPECTED(column_category != ColumnNotFound, + "[Internal ERROR] the type of column: " + column_name + " can not found."); + // Get data type and size + auto column_id = column_name_id_[column_name]; + *column_data_type = column_data_type_[column_id]; + *column_data_type_size = ColumnDataTypeSize[*column_data_type]; + *column_shape = column_shape_[column_id]; + + // Retrieve value from json + if (column_category == ColumnInRaw) { + RETURN_IF_NOT_OK(GetColumnFromJson(column_name, columns_json, data_ptr, n_bytes)); + *data = reinterpret_cast(data_ptr->get()); + return Status::OK(); + } + + // Retrieve value from blob + RETURN_IF_NOT_OK(GetColumnFromBlob(column_name, columns_blob, data, data_ptr, n_bytes)); + if (*data == nullptr) { + *data = reinterpret_cast(data_ptr->get()); + } + return Status::OK(); +} + +Status ShardColumn::GetColumnFromJson(const std::string &column_name, const json &columns_json, + std::unique_ptr *data_ptr, uint64_t *n_bytes) { + RETURN_UNEXPECTED_IF_NULL(n_bytes); + RETURN_UNEXPECTED_IF_NULL(data_ptr); + auto column_id = column_name_id_[column_name]; + auto column_data_type = column_data_type_[column_id]; + + // Initialize num bytes + *n_bytes = ColumnDataTypeSize[column_data_type]; + auto json_column_value = columns_json[column_name]; + CHECK_FAIL_RETURN_UNEXPECTED(json_column_value.is_string() || json_column_value.is_number(), + "[Internal ERROR] the value of column: " + column_name + + " should be string or number but got: " + json_column_value.dump()); + switch (column_data_type) { + case ColumnFloat32: { + return GetFloat(data_ptr, json_column_value, false); + } + case ColumnFloat64: { + return GetFloat(data_ptr, json_column_value, true); + } + case ColumnInt32: { + return GetInt(data_ptr, json_column_value); + } + case ColumnInt64: { + return GetInt(data_ptr, json_column_value); + } + default: { + // Convert string to c_str + std::string tmp_string; + if (json_column_value.is_string()) { + tmp_string = json_column_value.get(); + } else { + tmp_string = json_column_value.dump(); + } + *n_bytes = tmp_string.size(); + auto data = reinterpret_cast(common::SafeCStr(tmp_string)); + *data_ptr = std::make_unique(*n_bytes); + for (uint32_t i = 0; i < *n_bytes; i++) { + (*data_ptr)[i] = *(data + i); + } + break; + } + } + return Status::OK(); +} + +template +Status ShardColumn::GetFloat(std::unique_ptr *data_ptr, const json &json_column_value, + bool use_double) { + RETURN_UNEXPECTED_IF_NULL(data_ptr); + std::unique_ptr array_data = std::make_unique(1); + if (json_column_value.is_number()) { + array_data[0] = json_column_value; + } else { + // Convert string to float + try { + if (use_double) { + array_data[0] = json_column_value.get(); + } else { + array_data[0] = json_column_value.get(); + } + } catch (json::exception &e) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + json_column_value.dump() + + " to type float, " + std::string(e.what())); + } + } + + auto data = reinterpret_cast(array_data.get()); + *data_ptr = std::make_unique(sizeof(T)); + for (uint32_t i = 0; i < sizeof(T); i++) { + (*data_ptr)[i] = *(data + i); + } + return Status::OK(); +} + +template +Status ShardColumn::GetInt(std::unique_ptr *data_ptr, const json &json_column_value) { + RETURN_UNEXPECTED_IF_NULL(data_ptr); + std::unique_ptr array_data = std::make_unique(1); + int64_t temp_value; + bool less_than_zero = false; + + if (json_column_value.is_number_integer()) { + const json json_zero = 0; + if (json_column_value < json_zero) { + less_than_zero = true; + } + temp_value = json_column_value; + } else if (json_column_value.is_string()) { + std::string string_value = json_column_value; + try { + if (!string_value.empty() && string_value[0] == '-') { + temp_value = std::stoll(string_value); + less_than_zero = true; + } else { + temp_value = static_cast(std::stoull(string_value)); + } + } catch (std::invalid_argument &e) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " + + std::string(e.what())); + } catch (std::out_of_range &e) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " + + std::string(e.what())); + } + } else { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value should be type string or number but got: " + + json_column_value.dump()); + } + + if ((less_than_zero && temp_value < static_cast(std::numeric_limits::min())) || + (!less_than_zero && static_cast(temp_value) > static_cast(std::numeric_limits::max()))) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value: " + std::to_string(temp_value) + " is out of range."); + } + array_data[0] = static_cast(temp_value); + + auto data = reinterpret_cast(array_data.get()); + *data_ptr = std::make_unique(sizeof(T)); + for (uint32_t i = 0; i < sizeof(T); i++) { + (*data_ptr)[i] = *(data + i); + } + return Status::OK(); +} + +Status ShardColumn::GetColumnFromBlob(const std::string &column_name, const std::vector &columns_blob, + const unsigned char **data, std::unique_ptr *data_ptr, + uint64_t *const n_bytes) { + RETURN_UNEXPECTED_IF_NULL(data); + uint64_t offset_address = 0; + auto column_id = column_name_id_[column_name]; + RETURN_IF_NOT_OK(GetColumnAddressInBlock(column_id, columns_blob, n_bytes, &offset_address)); + auto column_data_type = column_data_type_[column_id]; + if (has_compress_blob_ && column_data_type == ColumnInt32) { + RETURN_IF_NOT_OK(UncompressInt(column_id, data_ptr, columns_blob, n_bytes, offset_address)); + } else if (has_compress_blob_ && column_data_type == ColumnInt64) { + RETURN_IF_NOT_OK(UncompressInt(column_id, data_ptr, columns_blob, n_bytes, offset_address)); + } else { + *data = reinterpret_cast(&(columns_blob[offset_address])); + } + + return Status::OK(); +} + +ColumnCategory ShardColumn::CheckColumnName(const std::string &column_name) { + auto it_column = column_name_id_.find(column_name); + if (it_column == column_name_id_.end()) { + return ColumnNotFound; + } + auto it_blob = blob_column_id_.find(column_name); + return it_blob == blob_column_id_.end() ? ColumnInRaw : ColumnInBlob; +} + +std::vector ShardColumn::CompressBlob(const std::vector &blob, int64_t *compression_size) { + // Skip if no compress columns + *compression_size = 0; + if (!CheckCompressBlob()) { + return blob; + } + + std::vector dst_blob; + uint64_t i_src = 0; + for (int64_t i = 0; i < num_blob_column_; i++) { + // Get column data type + auto src_data_type = column_data_type_[column_name_id_[blob_column_[i]]]; + auto int_type = src_data_type == ColumnInt32 ? kInt32Type : kInt64Type; + + // Compress and return is blob has 1 column only + if (num_blob_column_ == 1) { + dst_blob = CompressInt(blob, int_type); + *compression_size = static_cast(blob.size()) - static_cast(dst_blob.size()); + return dst_blob; + } + + // Just copy and continue if column dat type is not int32/int64 + uint64_t num_bytes = BytesBigToUInt64(blob, i_src, kInt64Type); + if (src_data_type != ColumnInt32 && src_data_type != ColumnInt64) { + dst_blob.insert(dst_blob.end(), blob.begin() + i_src, blob.begin() + i_src + kInt64Len + num_bytes); + i_src += kInt64Len + num_bytes; + continue; + } + + // Get column slice in source blob + std::vector blob_slice(blob.begin() + i_src + kInt64Len, blob.begin() + i_src + kInt64Len + num_bytes); + // Compress column + auto dst_blob_slice = CompressInt(blob_slice, int_type); + // Get new column size + auto new_blob_size = UIntToBytesBig(dst_blob_slice.size(), kInt64Type); + // Append new column size + dst_blob.insert(dst_blob.end(), new_blob_size.begin(), new_blob_size.end()); + // Append new column data + dst_blob.insert(dst_blob.end(), dst_blob_slice.begin(), dst_blob_slice.end()); + i_src += kInt64Len + num_bytes; + } + MS_LOG(DEBUG) << "Compress blob data from " << blob.size() << " to " << dst_blob.size() << "."; + *compression_size = static_cast(blob.size()) - static_cast(dst_blob.size()); + return dst_blob; +} + +vector ShardColumn::CompressInt(const vector &src_bytes, const IntegerType &int_type) { + uint64_t i_size = kUnsignedOne << static_cast(int_type); + // Get number of elements + uint64_t src_n_int = src_bytes.size() / i_size; + // Calculate bitmap size (bytes) + uint64_t bitmap_size = (src_n_int + kNumDataOfByte - 1) / kNumDataOfByte; + + // Initialize destination blob, more space than needed, will be resized + vector dst_bytes(kBytesOfColumnLen + bitmap_size + src_bytes.size(), 0); + + // Write number of elements to destination blob + vector size_by_bytes = UIntToBytesBig(src_n_int, kInt32Type); + for (uint64_t n = 0; n < kBytesOfColumnLen; n++) { + dst_bytes[n] = size_by_bytes[n]; + } + + // Write compressed int + uint64_t i_dst = kBytesOfColumnLen + bitmap_size; + for (uint64_t i = 0; i < src_n_int; i++) { + // Initialize destination data type + IntegerType dst_int_type = kInt8Type; + // Shift to next int position + uint64_t pos = i * (kUnsignedOne << static_cast(int_type)); + // Narrow down this int + int64_t i_n = BytesLittleToMinIntType(src_bytes, pos, int_type, &dst_int_type); + + // Write this int to destination blob + uint64_t u_n = *reinterpret_cast(&i_n); + auto temp_bytes = UIntToBytesLittle(u_n, dst_int_type); + for (uint64_t j = 0; j < (kUnsignedOne << static_cast(dst_int_type)); j++) { + dst_bytes[i_dst++] = temp_bytes[j]; + } + + // Update date type in bit map + dst_bytes[i / kNumDataOfByte + kBytesOfColumnLen] |= + (static_cast(dst_int_type) << (kDataTypeBits * (kNumDataOfByte - kUnsignedOne - (i % kNumDataOfByte)))); + } + // Resize destination blob + dst_bytes.resize(i_dst); + MS_LOG(DEBUG) << "Compress blob field from " << src_bytes.size() << " to " << dst_bytes.size() << "."; + return dst_bytes; +} + +Status ShardColumn::GetColumnAddressInBlock(const uint64_t &column_id, const std::vector &columns_blob, + uint64_t *num_bytes, uint64_t *shift_idx) { + RETURN_UNEXPECTED_IF_NULL(num_bytes); + RETURN_UNEXPECTED_IF_NULL(shift_idx); + if (num_blob_column_ == 1) { + *num_bytes = columns_blob.size(); + *shift_idx = 0; + return Status::OK(); + } + auto blob_id = blob_column_id_[column_name_[column_id]]; + + for (int32_t i = 0; i < blob_id; i++) { + *shift_idx += kInt64Len + BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type); + } + *num_bytes = BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type); + + (*shift_idx) += kInt64Len; + + return Status::OK(); +} + +template +Status ShardColumn::UncompressInt(const uint64_t &column_id, std::unique_ptr *const data_ptr, + const std::vector &columns_blob, uint64_t *num_bytes, uint64_t shift_idx) { + RETURN_UNEXPECTED_IF_NULL(data_ptr); + RETURN_UNEXPECTED_IF_NULL(num_bytes); + auto num_elements = BytesBigToUInt64(columns_blob, shift_idx, kInt32Type); + *num_bytes = sizeof(T) * num_elements; + + // Parse integer array + uint64_t i_source = shift_idx + kBytesOfColumnLen + (num_elements + kNumDataOfByte - 1) / kNumDataOfByte; + auto array_data = std::make_unique(num_elements); + + for (uint64_t i = 0; i < num_elements; i++) { + uint8_t iBitMap = columns_blob[shift_idx + kBytesOfColumnLen + i / kNumDataOfByte]; + uint64_t i_type = (iBitMap >> ((kNumDataOfByte - 1 - (i % kNumDataOfByte)) * kDataTypeBits)) & kDataTypeBitMask; + auto mr_int_type = static_cast(i_type); + int64_t i64 = BytesLittleToMinIntType(columns_blob, i_source, mr_int_type); + i_source += (kUnsignedOne << i_type); + array_data[i] = static_cast(i64); + } + + auto data = reinterpret_cast(array_data.get()); + *data_ptr = std::make_unique(*num_bytes); + // field is none. for example: numpy is null + if (*num_bytes == 0) { + return Status::OK(); + } + CHECK_FAIL_RETURN_UNEXPECTED(memcpy_s(data_ptr->get(), *num_bytes, data, *num_bytes) == 0, + "[Internal ERROR] Failed to call securec func [memcpy_s]"); + return Status::OK(); +} + +uint64_t ShardColumn::BytesBigToUInt64(const std::vector &bytes_array, const uint64_t &pos, + const IntegerType &i_type) { + uint64_t result = 0; + for (uint64_t i = 0; i < (kUnsignedOne << static_cast(i_type)); i++) { + result = (result << kBitsOfByte) + bytes_array[pos + i]; + } + return result; +} + +std::vector ShardColumn::UIntToBytesBig(uint64_t value, const IntegerType &i_type) { + uint64_t n_bytes = kUnsignedOne << static_cast(i_type); + std::vector result(n_bytes, 0); + for (uint64_t i = 0; i < n_bytes; i++) { + result[n_bytes - 1 - i] = value & std::numeric_limits::max(); + value >>= kBitsOfByte; + } + return result; +} + +std::vector ShardColumn::UIntToBytesLittle(uint64_t value, const IntegerType &i_type) { + uint64_t n_bytes = kUnsignedOne << static_cast(i_type); + std::vector result(n_bytes, 0); + for (uint64_t i = 0; i < n_bytes; i++) { + result[i] = value & std::numeric_limits::max(); + value >>= kBitsOfByte; + } + return result; +} + +int64_t ShardColumn::BytesLittleToMinIntType(const std::vector &bytes_array, const uint64_t &pos, + const IntegerType &src_i_type, IntegerType *dst_i_type) { + uint64_t u_temp = 0; + for (uint64_t i = 0; i < (kUnsignedOne << static_cast(src_i_type)); i++) { + u_temp = (u_temp << kBitsOfByte) + + bytes_array[pos + (kUnsignedOne << static_cast(src_i_type)) - kUnsignedOne - i]; + } + + int64_t i_out; + switch (src_i_type) { + case kInt8Type: { + i_out = (int8_t)(u_temp & std::numeric_limits::max()); + break; + } + case kInt16Type: { + i_out = (int16_t)(u_temp & std::numeric_limits::max()); + break; + } + case kInt32Type: { + i_out = (int32_t)(u_temp & std::numeric_limits::max()); + break; + } + case kInt64Type: { + i_out = (int64_t)(u_temp & std::numeric_limits::max()); + break; + } + default: { + i_out = 0; + } + } + + if (!dst_i_type) { + return i_out; + } + + if (i_out >= static_cast(std::numeric_limits::min()) && + i_out <= static_cast(std::numeric_limits::max())) { + *dst_i_type = kInt8Type; + } else if (i_out >= static_cast(std::numeric_limits::min()) && + i_out <= static_cast(std::numeric_limits::max())) { + *dst_i_type = kInt16Type; + } else if (i_out >= static_cast(std::numeric_limits::min()) && + i_out <= static_cast(std::numeric_limits::max())) { + *dst_i_type = kInt32Type; + } else { + *dst_i_type = kInt64Type; + } + return i_out; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 2989217b31a0d26711007f3d48e26f8913eff37a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:41:23 +0800 Subject: [PATCH 084/160] ADD file via upload --- .../mindrecord/include/meta/shard_index.cc | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc new file mode 100644 index 00000000000..73397b5bba0 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_index.h" + +namespace mindspore { +namespace mindrecord { +// table name for index +const char TABLENAME[] = "index_table"; + +Index::Index() : database_name_(""), table_name_(TABLENAME) {} + +void Index::AddIndexField(const int64_t &schemaId, const std::string &field) { + fields_.emplace_back(pair(schemaId, field)); +} + +// Get attribute list +std::vector> Index::GetFields() { return fields_; } +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 12e19b939f98746b105a31978d8a3e9d22165bd9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:41:44 +0800 Subject: [PATCH 085/160] ADD file via upload --- .../mindrecord/include/meta/shard_page.cc | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc new file mode 100644 index 00000000000..ba2292415f6 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_page.h" +#include "pybind11/pybind11.h" + +namespace mindspore { +namespace mindrecord { +json Page::GetPage() const { + json str_page; + str_page["page_id"] = page_id_; + str_page["shard_id"] = shard_id_; + str_page["page_type"] = page_type_; + str_page["page_type_id"] = page_type_id_; + str_page["start_row_id"] = start_row_id_; + str_page["end_row_id"] = end_row_id_; + if (row_group_ids_.size() == 0) { + json row_groups = json({}); + row_groups["id"] = 0; + row_groups["offset"] = 0; + str_page["row_group_ids"].push_back(row_groups); + } else { + for (const auto &rg : row_group_ids_) { + json row_groups = json({}); + row_groups["id"] = rg.first; + row_groups["offset"] = rg.second; + str_page["row_group_ids"].push_back(row_groups); + } + } + str_page["page_size"] = page_size_; + return str_page; +} + +void Page::DeleteLastGroupId() { + if (!row_group_ids_.empty()) { + page_size_ = row_group_ids_.back().second; + row_group_ids_.pop_back(); + } +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From c90aba15e128cff84b9ab93c241ff9c9556e6298 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:42:02 +0800 Subject: [PATCH 086/160] ADD file via upload --- .../include/meta/shard_pk_sample.cc | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc new file mode 100644 index 00000000000..3974128fd5b --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_pk_sample.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples) + : ShardCategory(category_field, num_elements, std::numeric_limits::max(), true), + shuffle_(false), + num_samples_(num_samples) {} + +ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, + int64_t num_samples) + : ShardCategory(category_field, num_elements, num_categories, true), shuffle_(false), num_samples_(num_samples) {} + +ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, + uint32_t seed, int64_t num_samples) + : ShardCategory(category_field, num_elements, num_categories, true), shuffle_(true), num_samples_(num_samples) { + shuffle_op_ = std::make_shared(seed, kShuffleSample); // do shuffle and replacement +} + +Status ShardPkSample::SufExecute(ShardTaskList &tasks) { + if (shuffle_ == true) { + RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From d8e9f76a50af13965c6b35d479c62f15b29a2c22 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:42:24 +0800 Subject: [PATCH 087/160] ADD file via upload --- .../mindrecord/include/meta/shard_sample.cc | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc new file mode 100644 index 00000000000..cbafe84cb8f --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc @@ -0,0 +1,176 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_sample.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +ShardSample::ShardSample(int64_t n) + : numerator_(0), + denominator_(0), + partition_id_(0), + no_of_samples_(n), + indices_({}), + sampler_type_(kCustomTopNSampler), + offset_(-1) {} + +ShardSample::ShardSample(int64_t num, int64_t den) + : numerator_(num), + denominator_(den), + partition_id_(0), + no_of_samples_(0), + indices_({}), + sampler_type_(kCustomTopPercentSampler), + offset_(-1) {} + +ShardSample::ShardSample(int64_t num, int64_t den, int64_t par, int64_t no_of_samples, int64_t offset) + : numerator_(num), + denominator_(den), + partition_id_(par), + no_of_samples_(no_of_samples), + indices_({}), + sampler_type_(kCustomTopPercentSampler), + offset_(offset) {} + +ShardSample::ShardSample(const std::vector &indices) + : numerator_(0), + denominator_(0), + partition_id_(0), + no_of_samples_(0), + indices_(indices), + sampler_type_(kSubsetSampler) {} + +ShardSample::ShardSample(const std::vector &indices, uint32_t seed) : ShardSample(indices) { + sampler_type_ = kSubsetRandomSampler; + shuffle_op_ = std::make_shared(seed); +} + +int64_t ShardSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (sampler_type_ == kCustomTopNSampler) { + return no_of_samples_; + } + + if (sampler_type_ == kCustomTopPercentSampler) { + if (dataset_size % denominator_ == 0) { + return dataset_size / denominator_ * numerator_; + } else { + return dataset_size / denominator_ * numerator_ + 1; + } + } + if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { + return indices_.size(); + } + return 0; +} + +Status ShardSample::UpdateTasks(ShardTaskList &tasks, int64_t taking) { + if (tasks.permutation_.empty()) { + ShardTaskList new_tasks; + auto total_no = tasks.sample_ids_.size(); + CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0, + "[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no)); + if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { + for (int64_t i = 0; i < indices_.size(); ++i) { + int64_t index = ((indices_[i] % total_no) + total_no) % total_no; + new_tasks.AssignTask(tasks, index); // different mod result between c and python + } + } else { + int64_t count = 0; + if (nums_per_shard_.empty()) { + for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { + if (no_of_samples_ != 0 && count == no_of_samples_) break; + new_tasks.AssignTask(tasks, i % total_no); // rounding up. if overflow, go back to start + count++; + } + } else { + // Get samples within a specific range + int64_t i = partition_id_ - 1 >= 0 ? nums_per_shard_[partition_id_ - 1] : 0; + for (; i < nums_per_shard_[partition_id_]; i++) { + if (no_of_samples_ != 0 && count == no_of_samples_) break; + new_tasks.AssignTask(tasks, i % total_no); + count++; + } + } + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } else { + ShardTaskList new_tasks; + int64_t total_no = tasks.permutation_.size(); + CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0, + "[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no)); + int64_t cnt = 0; + for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { + if (no_of_samples_ != 0 && cnt == no_of_samples_) break; + new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]); + cnt++; + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } + return Status::OK(); +} + +Status ShardSample::Execute(ShardTaskList &tasks) { + if (offset_ != -1) { + int64_t old_v = 0; + int64_t num_rows_ = tasks.sample_ids_.size(); + for (int64_t x = 0; x < denominator_; x++) { + int64_t samples_per_buffer_ = (num_rows_ + offset_) / denominator_; + int64_t remainder = (num_rows_ + offset_) % denominator_; + if (x < remainder) samples_per_buffer_++; + if (x < offset_) samples_per_buffer_--; + old_v += samples_per_buffer_; + // nums_per_shard_ is used to save the current shard's ending index + nums_per_shard_.push_back(old_v); + } + } + int no_of_categories = static_cast(tasks.categories); + int64_t total_no = tasks.sample_ids_.size(); + int64_t taking = 0; + if (sampler_type_ == kCustomTopNSampler) { // non sharding case constructor #1 + no_of_samples_ = std::min(no_of_samples_, total_no); + taking = no_of_samples_ - no_of_samples_ % no_of_categories; + } else if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { + CHECK_FAIL_RETURN_UNEXPECTED(static_cast(indices_.size()) <= total_no, + "Invalid input, indices size: " + std::to_string(indices_.size()) + + " should be less than or equal to database size: " + std::to_string(total_no) + "."); + } else { // constructor TopPercent + if (numerator_ > 0 && denominator_ > 0 && numerator_ <= denominator_) { + if (numerator_ == 1 && denominator_ > 1) { // sharding + taking = (total_no + denominator_ - 1) / denominator_; + } else { // non sharding + taking = total_no * numerator_ / denominator_; + taking -= (taking % no_of_categories); + } + } else { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] 'numerator_': " + std::to_string(numerator_) + + " should be positive and less than denominator_: " + std::to_string(denominator_) + "."); + } + } + return UpdateTasks(tasks, taking); +} + +Status ShardSample::SufExecute(ShardTaskList &tasks) { + if (sampler_type_ == kSubsetRandomSampler) { + RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 741349174fccdcfaf76ceef5f312f1c59cfa2203 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:42:44 +0800 Subject: [PATCH 088/160] ADD file via upload --- .../mindrecord/include/meta/shard_schema.cc | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc new file mode 100644 index 00000000000..d48d7694755 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc @@ -0,0 +1,163 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_schema.h" +#include "utils/ms_utils.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +std::shared_ptr Schema::Build(std::string desc, const json &schema) { + // validate check + if (!Validate(schema)) { + return nullptr; + } + + std::vector blob_fields = PopulateBlobFields(schema); + Schema object_schema; + object_schema.desc_ = std::move(desc); + object_schema.blob_fields_ = std::move(blob_fields); + object_schema.schema_ = schema; + object_schema.schema_id_ = -1; + return std::make_shared(object_schema); +} + +std::string Schema::GetDesc() const { return desc_; } + +json Schema::GetSchema() const { + json str_schema; + str_schema["desc"] = desc_; + str_schema["schema"] = schema_; + str_schema["blob_fields"] = blob_fields_; + return str_schema; +} + +void Schema::SetSchemaID(int64_t id) { schema_id_ = id; } + +int64_t Schema::GetSchemaID() const { return schema_id_; } + +std::vector Schema::GetBlobFields() const { return blob_fields_; } + +std::vector Schema::PopulateBlobFields(json schema) { + std::vector blob_fields; + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + json it_value = it.value(); + if ((it_value.size() == kInt2 && it_value.find("shape") != it_value.end()) || it_value["type"] == "bytes") { + blob_fields.emplace_back(it.key()); + } + } + return blob_fields; +} + +bool Schema::ValidateNumberShape(const json &it_value) { + if (it_value.find("shape") == it_value.end()) { + MS_LOG(ERROR) << "Invalid schema, 'shape' object can not found in " << it_value.dump() + << ". Please check the input schema."; + return false; + } + + auto shape = it_value["shape"]; + if (!shape.is_array()) { + MS_LOG(ERROR) << "Invalid schema, the value of 'shape' should be list format but got: " << it_value["shape"] + << ". Please check the input schema."; + return false; + } + + int num_negtive_one = 0; + for (const auto &i : shape) { + if (i == 0 || i < -1) { + MS_LOG(ERROR) << "Invalid schema, the element of 'shape' value should be -1 or greater than 0 but got: " << i + << ". Please check the input schema."; + return false; + } + if (i == -1) { + num_negtive_one++; + } + } + + if (num_negtive_one > 1) { + MS_LOG(ERROR) << "Invalid schema, only 1 variable dimension(-1) allowed in 'shape' value but got: " + << it_value["shape"] << ". Please check the input schema."; + return false; + } + + return true; +} + +bool Schema::Validate(json schema) { + if (schema.empty()) { + MS_LOG(ERROR) << "Invalid schema, schema is empty. Please check the input schema."; + return false; + } + + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + // make sure schema key name must be composed of '0-9' or 'a-z' or 'A-Z' or '_' + if (!ValidateFieldName(it.key())) { + MS_LOG(ERROR) << "Invalid schema, field name: " << it.key() + << "is not composed of '0-9' or 'a-z' or 'A-Z' or '_'. Please rename the field name in schema."; + return false; + } + + json it_value = it.value(); + if (it_value.find("type") == it_value.end()) { + MS_LOG(ERROR) << "Invalid schema, 'type' object can not found in field " << it_value.dump() + << ". Please add the 'type' object for field in schema."; + return false; + } + + if (kFieldTypeSet.find(it_value["type"]) == kFieldTypeSet.end()) { + MS_LOG(ERROR) << "Invalid schema, the value of 'type': " << it_value["type"] + << " is not supported.\nPlease modify the value of 'type' to 'int32', 'int64', 'float32', " + "'float64', 'string', 'bytes' in schema."; + return false; + } + + if (it_value.size() == kInt1) { + continue; + } + + if (it_value["type"] == "bytes" || it_value["type"] == "string") { + MS_LOG(ERROR) + << "Invalid schema, no other field can be added when the value of 'type' is 'string' or 'types' but got: " + << it_value.dump() << ". Please remove other fields in schema."; + return false; + } + + if (it_value.size() != kInt2) { + MS_LOG(ERROR) << "Invalid schema, the fields should be 'type' or 'type' and 'shape' but got: " << it_value.dump() + << ". Please check the schema."; + return false; + } + + if (!ValidateNumberShape(it_value)) { + return false; + } + } + + return true; +} + +bool Schema::operator==(const mindrecord::Schema &b) const { + if (this->GetDesc() != b.GetDesc() || this->GetSchema() != b.GetSchema()) { + return false; + } + return true; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From daa2a5f5e9d50b4afbfab9db7db330cfceb6a06c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:43:02 +0800 Subject: [PATCH 089/160] ADD file via upload --- .../include/meta/shard_sequential_sample.cc | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc new file mode 100644 index 00000000000..2de121e8c22 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_sequential_sample.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +ShardSequentialSample::ShardSequentialSample(int64_t n, int64_t offset) + : ShardSample(n), offset_(offset), per_(0.0f), per_offset_(0.0f) {} + +ShardSequentialSample::ShardSequentialSample(float per, float per_offset) + : ShardSample(0), offset_(0), per_(per), per_offset_(per_offset) {} + +int64_t ShardSequentialSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) { + return dataset_size; + } + if (per_ > kEpsilon && per_ <= 1.0f) { + return dataset_size * kEpsilon; + } + return std::min(static_cast(no_of_samples_), dataset_size); +} + +Status ShardSequentialSample::Execute(ShardTaskList &tasks) { + int64_t taking; + int64_t total_no = static_cast(tasks.sample_ids_.size()); + if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) { + taking = total_no; + } else if (per_ > kEpsilon && per_ <= 1.0f) { + taking = total_no * kEpsilon; + } else { + taking = std::min(static_cast(no_of_samples_), total_no); + } + + if (tasks.permutation_.empty()) { + ShardTaskList new_tasks; + total_no = static_cast(tasks.Size()); + for (int64_t i = offset_; i < taking + offset_; ++i) { + new_tasks.AssignTask(tasks, i % total_no); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } else { // shuffled + ShardTaskList new_tasks; + total_no = static_cast(tasks.permutation_.size()); + for (int64_t i = offset_; i < taking + offset_; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } + return Status::OK(); +} + +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 3ceefb6ec362506995713a583460f6957eac7b19 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:43:23 +0800 Subject: [PATCH 090/160] ADD file via upload --- .../include/meta/shard_statistics.cc | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc new file mode 100644 index 00000000000..07bcad01702 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc @@ -0,0 +1,94 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_statistics.h" +#include "pybind11/pybind11.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::ERROR; + +namespace mindspore { +namespace mindrecord { +std::shared_ptr Statistics::Build(std::string desc, const json &statistics) { + // validate check + if (!Validate(statistics)) { + return nullptr; + } + Statistics object_statistics; + object_statistics.desc_ = std::move(desc); + object_statistics.statistics_ = statistics; + object_statistics.statistics_id_ = -1; + return std::make_shared(object_statistics); +} + +std::string Statistics::GetDesc() const { return desc_; } + +json Statistics::GetStatistics() const { + json str_statistics; + str_statistics["desc"] = desc_; + str_statistics["statistics"] = statistics_; + return str_statistics; +} + +void Statistics::SetStatisticsID(int64_t id) { statistics_id_ = id; } + +int64_t Statistics::GetStatisticsID() const { return statistics_id_; } + +bool Statistics::Validate(const json &statistics) { + if (statistics.size() != kInt1) { + MS_LOG(ERROR) << "Invalid data, 'statistics' is empty."; + return false; + } + if (statistics.find("level") == statistics.end()) { + MS_LOG(ERROR) << "Invalid data, 'level' object can not found in statistic"; + return false; + } + return LevelRecursive(statistics["level"]); +} + +bool Statistics::LevelRecursive(json level) { + bool ini = true; + for (json::iterator it = level.begin(); it != level.end(); ++it) { + json a = it.value(); + if (a.size() == kInt2) { + if ((a.find("key") == a.end()) || (a.find("count") == a.end())) { + MS_LOG(ERROR) << "Invalid data, the node field is 2, but 'key'/'count' object does not existed"; + return false; + } + } else if (a.size() == kInt3) { + if ((a.find("key") == a.end()) || (a.find("count") == a.end()) || a.find("level") == a.end()) { + MS_LOG(ERROR) << "Invalid data, the node field is 3, but 'key'/'count'/'level' object does not existed"; + return false; + } else { + ini = LevelRecursive(a.at("level")); + } + } else { + MS_LOG(ERROR) << "Invalid data, the node field is not equal to 2 or 3"; + return false; + } + } + return ini; +} + +bool Statistics::operator==(const Statistics &b) const { + if (this->GetStatistics() != b.GetStatistics()) { + return false; + } + return true; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 8b6c2e82a796c0a93af5ed031674f9d0c1130295 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:43:42 +0800 Subject: [PATCH 091/160] ADD file via upload --- .../include/meta/shard_task_list.cc | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc new file mode 100644 index 00000000000..43a8e938794 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc @@ -0,0 +1,146 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/util/random.h" +#include "minddata/mindrecord/include/shard_task_list.h" +#include "utils/ms_utils.h" +#include "minddata/mindrecord/include/common/shard_utils.h" + +using mindspore::LogStream; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::MsLogLevel::DEBUG; + +namespace mindspore { +namespace mindrecord { +ShardTaskList::ShardTaskList() : categories(1) {} + +ShardTaskList::ShardTaskList(const ShardTaskList &other) + : categories(other.categories), + permutation_(other.permutation_), + sample_ids_(other.sample_ids_), + task_list_(other.task_list_) {} + +ShardTaskList &ShardTaskList::operator=(const ShardTaskList &other) { + ShardTaskList tmp(other); + std::swap(categories, tmp.categories); + permutation_.swap(tmp.permutation_); + sample_ids_.swap(tmp.sample_ids_); + task_list_.swap(tmp.task_list_); + return *this; +} + +void ShardTaskList::InitSampleIds() { + // no-op if there already exists sample ids. Do not clobber previous list + if (sample_ids_.empty()) { + sample_ids_ = std::vector(task_list_.size()); + for (auto i = 0; i < task_list_.size(); i++) { + sample_ids_[i] = i; + } + } +} + +void ShardTaskList::MakePerm() { + int64_t perm_size = sample_ids_.size(); + permutation_ = std::vector(perm_size); + for (int64_t i = 0; i < perm_size; i++) { + permutation_[i] = i; + } +} + +// Swap the new_tasks with orig_tasks +void ShardTaskList::TaskListSwap(ShardTaskList &orig_tasks, ShardTaskList &new_tasks) { + // When swapping, if the orig_tasks contains fields that need to be preserved after the swap, then swapping with a + // new_tasks that does not have those fields will result in clobbering/losing the data after the swap. + // The task_list_ should not be lost/clobbered. + // This function can be called in the middle of mindrecord's epoch, when orig_tasks.task_list_ is still being + // used by mindrecord op's worker threads. So don't touch its task_list_ since this field should be preserved anyways. + + std::swap(orig_tasks.categories, new_tasks.categories); + std::swap(orig_tasks.permutation_, new_tasks.permutation_); + std::swap(orig_tasks.sample_ids_, new_tasks.sample_ids_); +} + +void ShardTaskList::PopBack() { task_list_.pop_back(); } + +int64_t ShardTaskList::Size() const { return static_cast(task_list_.size()); } + +int64_t ShardTaskList::SizeOfRows() const { + if (task_list_.size() == 0) return static_cast(0); + + // 1 task is 1 page + const size_t kBlobInfoIndex = 2; + auto sum_num_rows = [](int64_t x, ShardTask y) { return x + std::get(y)[0]; }; + int64_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows); + return nRows; +} + +ShardTask &ShardTaskList::GetTaskByID(int64_t id) { return task_list_[id]; } + +int64_t ShardTaskList::GetTaskSampleByID(int64_t id) { return sample_ids_[id]; } + +int64_t ShardTaskList::GetRandomTaskID() { + std::mt19937 gen = mindspore::dataset::GetRandomDevice(); + std::uniform_int_distribution<> dis(0, sample_ids_.size() - 1); + return dis(gen); +} + +ShardTask &ShardTaskList::GetRandomTask() { + std::mt19937 gen = mindspore::dataset::GetRandomDevice(); + std::uniform_int_distribution<> dis(0, task_list_.size() - 1); + return task_list_[dis(gen)]; +} + +ShardTaskList ShardTaskList::Combine(std::vector &category_tasks, bool replacement, int64_t num_elements, + int64_t num_samples) { + ShardTaskList res; + if (category_tasks.empty()) return res; + auto total_categories = category_tasks.size(); + res.categories = static_cast(total_categories); + if (replacement == false) { + auto minTasks = category_tasks[0].Size(); + for (int64_t i = 1; i < total_categories; i++) { + minTasks = std::min(minTasks, category_tasks[i].Size()); + } + int64_t count = 0; + for (int64_t task_no = 0; task_no < minTasks; task_no++) { + for (int64_t i = 0; i < total_categories; i++) { + if (num_samples != 0 && count == num_samples) break; + res.InsertTask(std::move(category_tasks[i].GetTaskByID(task_no))); + count++; + } + } + } else { + auto maxTasks = category_tasks[0].Size(); + for (int64_t i = 1; i < total_categories; i++) { + maxTasks = std::max(maxTasks, category_tasks[i].Size()); + } + if (num_elements != std::numeric_limits::max()) { + maxTasks = static_cast(num_elements); + } + int64_t count = 0; + for (int64_t i = 0; i < total_categories; i++) { + for (int64_t j = 0; j < maxTasks; j++) { + if (num_samples != 0 && count == num_samples) break; + res.InsertTask(category_tasks[i].GetRandomTask()); + count++; + } + } + } + + return res; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 17df3cb7b6d71956b3a20de4383d9e08c3e6d8fa Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sun, 16 Oct 2022 22:57:13 +0800 Subject: [PATCH 092/160] ADD file via upload --- .../mindrecord/include/meta/shard_shuffle.cc | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc new file mode 100644 index 00000000000..4dd331b3316 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc @@ -0,0 +1,203 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_shuffle.h" + +#include + +namespace mindspore { +namespace mindrecord { +ShardShuffle::ShardShuffle(uint32_t seed, ShuffleType shuffle_type) + : shuffle_seed_(seed), + no_of_samples_(0), + replacement_(false), + reshuffle_each_epoch_(true), + shuffle_type_(shuffle_type) {} + +ShardShuffle::ShardShuffle(uint32_t seed, int64_t no_of_samples, bool replacement, bool reshuffle_each_epoch, + ShuffleType shuffle_type) + : shuffle_seed_(seed), + no_of_samples_(no_of_samples), + replacement_(replacement), + reshuffle_each_epoch_(reshuffle_each_epoch), + shuffle_type_(shuffle_type) {} + +int64_t ShardShuffle::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (replacement_) { + return no_of_samples_ == 0 ? dataset_size : no_of_samples_; + } + return no_of_samples_ == 0 ? dataset_size : std::min(dataset_size, no_of_samples_); +} + +Status ShardShuffle::CategoryShuffle(ShardTaskList &tasks) { + int64_t individual_size = tasks.sample_ids_.size() / tasks.categories; + std::vector> new_permutations(tasks.categories, std::vector(individual_size)); + for (int64_t i = 0; i < tasks.categories; i++) { + for (int64_t j = 0; j < individual_size; j++) new_permutations[i][j] = j; + std::shuffle(new_permutations[i].begin(), new_permutations[i].end(), std::default_random_engine(shuffle_seed_)); + } + tasks.permutation_.clear(); + for (int64_t j = 0; j < individual_size; j++) { + for (int64_t i = 0; i < tasks.categories; i++) { + tasks.permutation_.push_back(new_permutations[i][j] * tasks.categories + i); + } + } + + ShardTaskList new_tasks; + for (int64_t i = 0; i < individual_size; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + + return Status::OK(); +} + +Status ShardShuffle::ShuffleFiles(ShardTaskList &tasks) { + if (no_of_samples_ == 0) { + no_of_samples_ = tasks.Size(); + } + CHECK_FAIL_RETURN_UNEXPECTED( + no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_)); + auto shard_sample_cout = GetShardSampleCount(); + + // shuffle the files index + std::vector shuffle_files; + for (int64_t i = 0; i < shard_sample_cout.size(); i++) { + shuffle_files.push_back(i); + } + std::shuffle(shuffle_files.begin(), shuffle_files.end(), std::default_random_engine(shuffle_seed_)); + + // reconstruct the permutation between files + // -- before -- + // file1: [0, 1, 2] + // file2: [3, 4, 5, 6] + // file3: [7, 8] + // file4: [9, 10] + // files: [file1, file2, file3, file4] + // permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + // -- after -- + // files: [file4, file1, file3, file2] + // permutation : [9, 10, 0, 1, 2, 7, 8, 3, 4, 5, 6] + auto original_permutation = tasks.permutation_; + int64_t whole_index = 0; + for (int64_t i = 0; i < shuffle_files.size(); i++) { + int64_t start_index = 0; + int64_t current_size = 0; + if (shuffle_files[i] == 0) { + start_index = 0; + current_size = shard_sample_cout[shuffle_files[i]]; + } else { + start_index = shard_sample_cout[shuffle_files[i] - 1]; + current_size = shard_sample_cout[shuffle_files[i]] - start_index; + } + std::copy(original_permutation.begin() + start_index, original_permutation.begin() + start_index + current_size, + tasks.permutation_.begin() + whole_index); + whole_index += current_size; + } + + auto total_no = tasks.Size(); + int64_t samples_to_assign = + (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); + ShardTaskList new_tasks; + for (int64_t i = 0; i < samples_to_assign; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + return Status::OK(); +} + +Status ShardShuffle::ShuffleInfile(ShardTaskList &tasks) { + if (no_of_samples_ == 0) { + no_of_samples_ = tasks.Size(); + } + CHECK_FAIL_RETURN_UNEXPECTED( + no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_)); + // reconstruct the permutation in file + // -- before -- + // file1: [0, 1, 2] + // file2: [3, 4, 5, 6] + // file3: [7, 8] + // file4: [9, 10] + // files: [file1, file2, file3, file4] + // permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + // -- after -- + // permutation: [2, 0, 1, 4, 6, 3, 5, 8, 7, 9, 10] + auto shard_sample_cout = GetShardSampleCount(); + int64_t start_index = 0; + for (int64_t i = 0; i < shard_sample_cout.size(); i++) { + auto current_size = shard_sample_cout[i] - start_index; + std::shuffle(tasks.permutation_.begin() + start_index, tasks.permutation_.begin() + start_index + current_size, + std::default_random_engine(shuffle_seed_)); + start_index = shard_sample_cout[i]; + } + auto total_no = tasks.Size(); + ShardTaskList new_tasks; + int64_t samples_to_assign = + (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); + for (int64_t i = 0; i < samples_to_assign; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + return Status::OK(); +} + +Status ShardShuffle::Execute(ShardTaskList &tasks) { + if (reshuffle_each_epoch_) { + shuffle_seed_++; + } + CHECK_FAIL_RETURN_UNEXPECTED(tasks.categories >= 1, + "[Internal ERROR] task categories should be greater than or equal to 1 but got: " + + std::to_string(tasks.categories)); + if (shuffle_type_ == kShuffleSample) { // shuffle each sample + if (tasks.permutation_.empty() == true) { + tasks.MakePerm(); + } + if (GetShuffleMode() == dataset::ShuffleMode::kGlobal) { + if (replacement_ == true) { + ShardTaskList new_tasks; + if (no_of_samples_ == 0) { + no_of_samples_ = tasks.sample_ids_.size(); + } + CHECK_FAIL_RETURN_UNEXPECTED(no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + + std::to_string(no_of_samples_)); + for (uint32_t i = 0; i < no_of_samples_; ++i) { + new_tasks.AssignTask(tasks, tasks.GetRandomTaskID()); + } + + ShardTaskList::TaskListSwap(tasks, new_tasks); + } else { + std::shuffle(tasks.permutation_.begin(), tasks.permutation_.end(), std::default_random_engine(shuffle_seed_)); + auto total_no = tasks.Size(); + ShardTaskList new_tasks; + int64_t samples_to_assign = + (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); + for (int64_t i = 0; i < samples_to_assign; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } + } else if (GetShuffleMode() == dataset::ShuffleMode::kInfile) { + RETURN_IF_NOT_OK(ShuffleInfile(tasks)); + } else if (GetShuffleMode() == dataset::ShuffleMode::kFiles) { + RETURN_IF_NOT_OK(ShuffleFiles(tasks)); + } + } else { // shuffle unit like: (a1, b1, c1),(a2, b2, c2),..., (an, bn, cn) + return this->CategoryShuffle(tasks); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From c47fd2ea5a458dc988f18951bde4987347a16c11 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:26:45 +0800 Subject: [PATCH 093/160] ADD file via upload --- .../ccsrc/transform/graph_ir/all_ops.h | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/all_ops.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/all_ops.h b/mindspore2022/ccsrc/transform/graph_ir/all_ops.h new file mode 100644 index 00000000000..d815784e7b9 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/all_ops.h @@ -0,0 +1,22 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_ALL_OPS_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_ALL_OPS_H_ + +// old +#include "ops/all_ops.h" +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_ALL_OPS_H_ -- 2.34.1 From fcc584b8e23412aca91ab52310a45bb287088fe6 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:27:05 +0800 Subject: [PATCH 094/160] ADD file via upload --- mindspore2022/ccsrc/transform/graph_ir/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/CMakeLists.txt diff --git a/mindspore2022/ccsrc/transform/graph_ir/CMakeLists.txt b/mindspore2022/ccsrc/transform/graph_ir/CMakeLists.txt new file mode 100644 index 00000000000..294739d0075 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/CMakeLists.txt @@ -0,0 +1,8 @@ +if(ENABLE_D OR ENABLE_ACL) + file(GLOB_RECURSE _TRANSFORM_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") + list(REMOVE_ITEM _TRANSFORM_SRC_LIST "graph_ir/op_declare/hcom_ops_declare.cc") + set_property(SOURCE ${_TRANSFORM_SRC_LIST} PROPERTY COMPILE_DEFINITIONS + SUBMODULE_ID=mindspore::SubModuleId::SM_GE_ADPT) + add_library(_mindspore_transform_graph_ir_obj OBJECT ${_TRANSFORM_SRC_LIST}) + +endif() -- 2.34.1 From 5194daa7dbe453e01562a59f688bb459c273f5dd Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:27:21 +0800 Subject: [PATCH 095/160] ADD file via upload --- .../ccsrc/transform/graph_ir/convert.cc | 2031 +++++++++++++++++ 1 file changed, 2031 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/convert.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/convert.cc b/mindspore2022/ccsrc/transform/graph_ir/convert.cc new file mode 100644 index 00000000000..1f1e5e3a704 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/convert.cc @@ -0,0 +1,2031 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/convert.h" + +#include +#include +#include +#include "include/common/utils/utils.h" + +#include "base/core_ops.h" +#include "frontend/operator/ops.h" +#include "utils/log_adapter.h" +#include "ir/graph_utils.h" +#include "utils/symbolic.h" +#include "include/common/utils/config_manager.h" +#include "include/common/utils/convert_utils.h" +#include "utils/ms_context.h" +#include "utils/check_convert_utils.h" +#include "include/transform/graph_ir/op_adapter_map.h" +#include "ops/state_ops.h" +#include "ops/array_ops.h" +#include "ops/elewise_calculation_ops.h" +#include "ops/math_ops.h" +#ifdef ENABLE_D +#include "ops/save_ops.h" +#endif +#include "transform/graph_ir/op_adapter.h" +#include "transform/graph_ir/op_adapter_desc.h" + +namespace mindspore { +namespace transform { +using std::endl; + +using ge::Operator; +using mindspore::kAnyValue; +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; +using Variable = ge::op::Variable; +using Constant = ge::op::Constant; +using Assign = ge::op::Assign; +using Data = ge::op::Data; + +namespace { +std::vector GetOrderedCNodes(const FuncGraphPtr fg) { + MS_EXCEPTION_IF_NULL(fg); + auto BelongSameGraph = std::bind(IncludeBelongGraph, fg, std::placeholders::_1); + auto succ_include_fv = [&fg](const AnfNodePtr &node) -> std::vector { + std::vector vecs; + if (node == nullptr) { + return vecs; + } + if (node->isa()) { + auto cnode = node->cast(); + auto &inputs = cnode->inputs(); + // Check if free variables used. + for (const auto &input : inputs) { + auto input_fg = GetValueNode(input); + if (input_fg) { + for (auto &fv : input_fg->free_variables_nodes()) { + if (fv->func_graph() == fg && fg->nodes().contains(fv)) { + vecs.push_back(fv); + } + } + } + } + (void)vecs.insert(vecs.end(), inputs.begin(), inputs.end()); + } + return vecs; + }; + + return TopoSort(fg->get_return(), succ_include_fv, BelongSameGraph); +} +} // namespace + +// ---------------implement of DfGraphConvertor------------- +bool IsCaseNode(const CNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + if (!node->inputs().empty() && node->input(0)->isa() && + GetCNodeFuncName(node->input(0)->cast()) == "switch_layer") { + return true; + } + return false; +} + +std::string GetCNodeTargetFuncName(const CNodePtr cnode) { + if (IsCaseNode(cnode)) { + return string(kNameCase); + } + auto name = GetCNodeFuncName(cnode); + if (name == "switch_layer") { + name = ""; + } + return name; +} + +OpAdapterPtr DfGraphConvertor::FindAdapter(const AnfNodePtr node, bool train) { + MS_EXCEPTION_IF_NULL(node); + if (node->isa()) { + auto cnode = node->cast(); + + std::string name = kNameCustomOp; + if (!IsCustomCNode(cnode)) { + name = GetCNodeTargetFuncName(cnode); + } + + auto it_adpt = OpAdapterMap::get().find(name); + if (it_adpt != OpAdapterMap::get().end()) { + return it_adpt->second->Get(train); + } + MS_LOG(EXCEPTION) << "Can't find OpAdapter for " << name; + } + + if (node->isa()) { + return OpAdapterMap::get()[kNameConst]->Get(train); + } + if (node->isa()) { + return OpAdapterMap::get()[kNameParam]->Get(train); + } + return OpAdapterPtr(nullptr); +} + +void DfGraphConvertor::InitLoopVar(std::vector *init_input) { + MS_EXCEPTION_IF_NULL(init_input); + if (this->training_) { + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64); + auto var_iter_num = std::make_shared("npu_runconfig/iterations_per_loop"); + auto var_loop_cond = std::make_shared("npu_runconfig/loop_cond"); + auto var_one = std::make_shared("npu_runconfig/one"); + auto var_zero = std::make_shared("npu_runconfig/zero"); + (void)var_iter_num->update_output_desc_y(desc); + (void)var_loop_cond->update_output_desc_y(desc); + (void)var_one->update_output_desc_y(desc); + (void)var_zero->update_output_desc_y(desc); + vars_["npu_runconfig/iterations_per_loop"] = var_iter_num; + vars_["npu_runconfig/loop_cond"] = var_loop_cond; + vars_["npu_runconfig/one"] = var_one; + vars_["npu_runconfig/zero"] = var_zero; + + int64_t value = 0; + auto const_iter_num = std::make_shared("const/npu_runconfig/iterations_per_loop"); + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + value = ConfigManager::GetInstance().iter_num(); + } else { + MS_LOG(INFO) << "Run with normal(non-sink) mode, the iterator number will always be 1"; + ConfigManager::GetInstance().ResetIterNum(); + } + value -= 1; // iteration start from 0, the max iteration number for n loop should be n-1 + (void)const_iter_num->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_loop_cond = std::make_shared("const/npu_runconfig/loop_cond"); + value = 0; + (void)const_loop_cond->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_one = std::make_shared("const/npu_runconfig/one"); + value = 1; + (void)const_one->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_zero = std::make_shared("const/npu_runconfig/zero"); + value = 0; + (void)const_zero->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + (void)const_iter_num->update_output_desc_y(desc); + (void)const_loop_cond->update_output_desc_y(desc); + (void)const_one->update_output_desc_y(desc); + (void)const_zero->update_output_desc_y(desc); + + auto assign_iter_num = std::make_shared("assign/npu_runconfig/iterations_per_loop"); + (void)assign_iter_num->set_input_ref(*var_iter_num).set_input_value(*const_iter_num); + auto assign_loop_cond = std::make_shared("assign/npu_runconfig/loop_cond"); + (void)assign_loop_cond->set_input_ref(*var_loop_cond).set_input_value(*const_loop_cond); + auto assign_one = std::make_shared("assign/npu_runconfig/one"); + (void)assign_one->set_input_ref(*var_one).set_input_value(*const_one); + auto assign_zero = std::make_shared("assign/npu_runconfig/zero"); + (void)assign_zero->set_input_ref(*var_zero).set_input_value(*const_zero); + + init_input->push_back(*var_iter_num); + init_input->push_back(*var_loop_cond); + init_input->push_back(*var_one); + init_input->push_back(*var_zero); + init_ops_.push_back(var_iter_num); + init_ops_.push_back(var_loop_cond); + init_ops_.push_back(var_one); + init_ops_.push_back(var_zero); + init_ops_.push_back(const_iter_num); + init_ops_.push_back(const_loop_cond); + init_ops_.push_back(const_one); + init_ops_.push_back(const_zero); + init_ops_.push_back(assign_iter_num); + init_ops_.push_back(assign_loop_cond); + init_ops_.push_back(assign_one); + init_ops_.push_back(assign_zero); + } +} + +OpAdapterPtr DfGraphConvertor::FindAdapter(const std::string &name, bool train) { + auto it = OpAdapterMap::get().find(name); + if (it != OpAdapterMap::get().end()) { + return it->second->Get(train); + } + MS_LOG(EXCEPTION) << "Can't find OpAdapter for " << name; +} + +void DfGraphConvertor::DrawParamInitSubGraph(const std::string &name, const AnfNodePtr &it) { + // draw init subgraph + init_sout_ << "op_assign" << it.get() << "[label=<"; + init_sout_ << "" << endl; + init_sout_ << ""; + init_sout_ << ""; + init_sout_ << ""; + init_sout_ << "" << endl; + init_sout_ << "" << endl; + init_sout_ << "
resourcevalue
" + << "\"assign_" << name << "\"
> shape=plaintext]" << endl; + init_sout_ << "param" << it.get() << "[shape=octagon, label=\"" << name << "\"]" << endl; + init_sout_ << "const" << it.get() << "[label= \"" << name << "_const" + << "\" shape=ellipse]" << endl; + init_sout_ << "param" << it.get() << "->" + << "op_assign" << it.get() << ":1" << endl; + init_sout_ << "const" << it.get() << "->" + << "op_assign" << it.get() << ":2" << endl; +} + +void DfGraphConvertor::SetupParamInitSubGraph(const TensorOrderMap &tensors, std::vector *init_input) { + DfGraphPtr init_graph = std::make_shared("init"); + std::vector nodes = GetOrderedCNodes(anf_graph_); + + for (auto &it : nodes) { + MS_EXCEPTION_IF_NULL(it); + if (it->isa()) { + if (IsValueNode(it)) { + auto symbolic = GetValueNode(it); + auto name = std::static_pointer_cast(symbolic->node())->name(); + auto iter = vars_.find(name); // get corresponding variable op + if (iter != vars_.end()) { + op_cache_[it.get()] = iter->second; + // #ifdef DRAW_GE_GRAPH + compute_sout_ << op_draw_name_[params_[name].get()] << " -> " << op_draw_name_[it.get()] + << "[style=\"dotted\"]" << endl; + // #endif + } + } else if (IsValueNode(it)) { + auto refkey = GetValueNode(it); + MS_EXCEPTION_IF_NULL(refkey); + auto name = refkey->tag(); + auto iter = vars_.find(name); // get corresponding variable op + if (iter != vars_.end()) { + op_cache_[it.get()] = iter->second; + compute_sout_ << op_draw_name_[params_[name].get()] << " -> " << op_draw_name_[it.get()] + << "[style=\"dotted\"]" << endl; + } + } + } + } + + for (auto &it : tensors) { + if (vars_.find(it.first) == vars_.end()) { + MS_LOG(WARNING) << "Init parameter " << it.first << " didn't appear in graph."; + vars_[it.first] = nullptr; + } + } + + // set up init sub graph + if (init_input->size()) { + // init sub graph needs no input + MS_LOG(INFO) << "Build data init subgraph."; + (void)init_graph->SetInputs(*init_input); + this->init_graph_ = init_graph; + } else { + this->init_graph_ = nullptr; + } +} + +void DfGraphConvertor::MakeDatasetHandler(const std::string &name, const size_t &input_idx, const AnfNodePtr &it) { + MS_LOG(INFO) << "The " << name << " is the " << input_idx << "(st/nd/th) input"; + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + auto getnext_idx = static_cast(input_idx); + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + if (!param.input_indexes().empty() && input_idx <= param.input_indexes().size()) { + getnext_idx = param.input_indexes()[input_idx] - 1; // input_idx start from 0. + MS_LOG(INFO) << "remap input_index:" << input_idx << " to getnext_index:" << getnext_idx << "."; + } + // use iterator_getnext op with output_name instead of data op in BuildGraph. + if (dataset_iter_getnext_ != nullptr) { + out_handle_cache_[it.get()] = OutHandler(dataset_iter_getnext_, "y" + std::to_string(getnext_idx)); + } + } +} + +void DfGraphConvertor::SetupBroadcast(const std::shared_ptr &broadcast, + const std::vector &broadcast_desc, + const DfGraphPtr &broadcast_graph, std::vector broadcast_input) { + MS_LOG(INFO) << "build broadcast subgraph"; + if (broadcast_desc.size() != broadcast_input.size()) { + MS_LOG(EXCEPTION) << "Desc number of BroadCast is not equal to number of Input"; + } + (void)broadcast->create_dynamic_input_x(static_cast(broadcast_input.size())); + (void)broadcast->create_dynamic_output_y(static_cast(broadcast_desc.size())); + for (unsigned int i = 0; i < broadcast_input.size(); i++) { + (void)broadcast->set_dynamic_input_x(i, broadcast_input[i]); + (void)broadcast->update_dynamic_output_desc_y(i, broadcast_desc[i]); + } + (void)broadcast_graph->SetInputs(broadcast_input); + this->broadcast_graph_ = broadcast_graph; +} + +void DfGraphConvertor::InitParamWithData(const TensorOrderMap &tensors) { + int index = 0; + std::vector init_input; + for (auto it : tensors) { + std::string name = it.first; + auto node_itor = params_.find(name); + // if name not in params_, create a node in graph + if (node_itor == params_.end()) { + MS_LOG(WARNING) << name << " is not in params, and create a new node."; + ParameterPtr param = std::make_shared(nullptr); + name = name + "_temp"; + param->set_name(name); + (void)ConvertParameter(param); + node_itor = params_.find(name); + } + auto node = node_itor->second; + auto op_itor = op_cache_.find(node.get()); + if (op_itor == op_cache_.end()) { + MS_LOG(EXCEPTION) << "Can not find op for node " << node->ToString() << "."; + } + auto adpt = FindAdapter(kNameParam, training_); + if (adpt == nullptr) continue; + auto param_op = adpt->generate(name + "_data"); + MS_LOG(INFO) << "Add parameter " << name << " as input, index " << index << "."; + + if (!training_) { + auto adpt_const = FindAdapter(kNameConst, training_); + if (adpt_const == nullptr) continue; + auto const_op = adpt_const->generate(name + "_const"); + (void)adpt_const->setAttr(const_op, "value", it.second); + + auto const_op_desc = TransformUtil::GetGeTensorDesc(it.second->shape_c(), it.second->data_type(), kOpFormat_NCHW); + if (const_op_desc == nullptr) { + MS_LOG(WARNING) << "Create variable " << name << " output descriptor failed!"; + continue; + } + (void)std::static_pointer_cast(const_op)->update_output_desc_y(*const_op_desc); + + vars_[name] = const_op; + op_itor->second = const_op; + continue; + } + + // create tensor descriptor for output descriptor + auto desc = TransformUtil::GetGeTensorDesc(it.second->shape_c(), it.second->data_type(), kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " output descriptor failed!"; + continue; + } + + // we need three variable ops for each graph with same name + // build init subgraph + if (it.second->is_init() == 0) { + (void)std::static_pointer_cast(param_op)->set_attr_index(index++); + auto init_var = std::make_shared(name); + auto assign_op = std::make_shared("assign_" + name); + (void)init_var->update_output_desc_y(*desc); + (void)assign_op->set_input_ref(*init_var).set_input_value(*param_op); + init_input.push_back(*init_var); + init_ops_.push_back(param_op); + init_ops_.push_back(assign_op); + init_ops_.push_back(init_var); + } + + auto variable = std::make_shared(name); + (void)variable->update_output_desc_y(*desc); + // do not use read variable while variable sink + MS_LOG(DEBUG) << "InitParam, op_name = " << name << ", var = " << variable->GetName() << "."; + op_itor->second = variable; // replace parameter with variable + vars_[name] = variable; // prevent the variable operator from being freed + DrawParamInitSubGraph(name, node); + } + InitLoopVar(&init_input); + SetupParamInitSubGraph(tensors, &init_input); +} + +// convert all parameter need initialize to variable +DfGraphConvertor &DfGraphConvertor::InitParam(const TensorOrderMap &tensors) { + size_t input_idx = 0; + if (error_ != SUCCESS) { + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + error_ = INVALID_ARGUMENT; + MS_LOG(ERROR) << "Invalid AnfGraph in InitParam."; + return *this; + } + + // Processing input with MakeDatasetHandler + for (auto &it : anf_graph_->parameters()) { + auto op_itor = op_cache_.find(it.get()); // converted node + if (it->isa() && op_itor != op_cache_.end()) { + string name = std::static_pointer_cast(it)->name(); + auto tensor_itor = tensors.find(name); // in init value map + if (tensor_itor == tensors.end()) { + DfGraphConvertor::MakeDatasetHandler(name, input_idx, it); + input_idx++; + } + } + } + InitParamWithData(tensors); + init_sout_ << "}" << endl; + return *this; +} + +#if (defined ENABLE_D) +void DfGraphConvertor::BuildSaveCheckpointGraph() { + std::vector graph_inputs; + ge::op::Save save_op("save_parms"); + int save_op_is_active = 0; + size_t index = 0; + string name; + + auto count_size = std::count_if(vars_.begin(), vars_.end(), [](const auto &it) { + return LongToUlong(it.second == nullptr || it.first.find("/") != std::string::npos); + }); + + (void)save_op.create_dynamic_input_tensors(static_cast(vars_.size() - static_cast(count_size))); + + // for each "parameter" in anf graph excluding "input" + for (const auto &it : vars_) { + name = it.first; + if (it.second == nullptr || name.find("/") != std::string::npos) continue; + Variable variable(name); + (void)variable.update_output_desc_y(it.second->GetOutputDesc(0)); + (void)save_op.set_dynamic_input_tensors(static_cast(index++), variable); + + graph_inputs.push_back(variable); + + if (save_op_is_active == 0) { + checkpoint_sout_ << "op_save" << &save_op << "[label=<"; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "
tensor
" + << "\"saveop" + << "\"
> shape=plaintext]" << endl; + } + + checkpoint_sout_ << "param" << it.second << "[shape=octagon, label=\"" << name << "\"]" << endl; + + checkpoint_sout_ << "param" << it.second << "->" + << "op_save" << &save_op << ":1" << endl; + save_op_is_active = 1; + } + if (save_op_is_active) { + std::vector graph_output; + graph_output.emplace_back(save_op); + DfGraphPtr checkpoint_graph = std::make_shared("checkpoint"); + (void)checkpoint_graph->SetInputs(graph_inputs); + (void)checkpoint_graph->SetOutputs(graph_output); + this->save_ckp_graph_ = checkpoint_graph; + } else { + this->save_ckp_graph_ = nullptr; + } + + checkpoint_sout_ << "}" << endl; + return; +} +#endif + +DfGraphConvertor &DfGraphConvertor::GenerateBroadcastGraph(const TensorOrderMap &tensors) { + if (error_ != SUCCESS) { + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + error_ = INVALID_ARGUMENT; + MS_LOG(ERROR) << "Invalid AnfGraph in generate broadcast graph"; + return *this; + } + + DfGraphPtr broadcast_graph = std::make_shared("broadcast"); + // collect the operators create for broadcast sub graph, in order to avoid auto release + std::vector broadcast_input; + std::vector broadcast_desc; + auto broadcast = std::make_shared("broadcast_parameter"); + (void)broadcast->set_attr_root_rank(0); + (void)broadcast->set_attr_group("hccl_world_group"); + broadcast_ops_.push_back(broadcast); + + // find every parameter, build broadcast subgraph (or initialize the parameter with constant) + for (auto &it : anf_graph_->parameters()) { + auto op_itor = op_cache_.find(it.get()); // converted node + if (it->isa() && op_itor != op_cache_.end()) { + string name = std::static_pointer_cast(it)->name(); + auto tensor_itor = tensors.find(name); // in init tensor map + if (tensor_itor != tensors.end()) { + auto tensor = tensor_itor->second; + auto shape_ge = tensor->shape_c(); + + // create tensor descriptor for output descriptor + auto desc = TransformUtil::GetGeTensorDesc(shape_ge, tensor->data_type(), kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " output descriptor failed!"; + continue; + } + + // build broadcast subgraph + if (distribute_) { + auto broadcast_var = std::make_shared(name); + (void)broadcast_var->update_output_desc_y(*desc); + broadcast_input.push_back(*broadcast_var); + broadcast_desc.push_back(*desc); + broadcast_ops_.push_back(broadcast_var); + } + } + } + } + + // set up broadcast sub graph + if (!broadcast_input.empty()) { + DfGraphConvertor::SetupBroadcast(broadcast, broadcast_desc, broadcast_graph, broadcast_input); + } else { + this->broadcast_graph_ = nullptr; + } + return *this; +} + +DfGraphConvertor &DfGraphConvertor::GenerateCheckpointGraph() { + if (error_ != SUCCESS) { + MS_LOG(ERROR) << "Generate checkpoint graph failed, found error code " << error_ << "."; + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + error_ = INVALID_ARGUMENT; + MS_LOG(ERROR) << "Invalid AnfGraph in GenerateCheckpointGraph"; + return *this; + } +#ifdef ENABLE_D + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + BuildSaveCheckpointGraph(); + // Restoring from checkpoint file is done by pyfront, not in graph now. + } +#endif + return *this; +} + +DfGraphConvertor &DfGraphConvertor::ConvertAllNode() { + if (error_ != SUCCESS) { + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { + MS_LOG(ERROR) << "Invalid AnfGraph"; + error_ = FAILED; + return *this; + } + + compute_sout_.clear(); + compute_sout_ << "digraph {" << endl; + init_sout_.clear(); + init_sout_ << "digraph {" << endl; +#ifdef ENABLE_D + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + checkpoint_sout_.clear(); + checkpoint_sout_ << "digraph {" << endl; + } +#endif + restore_checkpoint_sout_.clear(); + restore_checkpoint_sout_ << "digraph {" << endl; + + // Convert all anf node to Operator + MS_LOG(DEBUG) << "convert all node"; + std::vector nodes = GetOrderedCNodes(anf_graph_); + for (auto &it : nodes) { + (void)Convert(it); + if (this->error_ != SUCCESS) { + MS_LOG(ERROR) << "failed to convert node: " << it->DebugString() << "."; + } + } + + // Create dataset iterator and iterator_getnext node + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + MS_LOG(INFO) << "Dataset param is " << param.ToString() << "."; + // GetNext + auto iter_getnext_op = make_shared("get_next_tmp"); + std::vector getnext_types; + const auto &origin_ge_types = param.ge_types(); + (void)std::transform( + origin_ge_types.begin(), origin_ge_types.end(), std::back_inserter(getnext_types), + [](int64_t t_num) -> enum ge::DataType { return static_cast(t_num); }); + (void)iter_getnext_op->set_attr_output_types(getnext_types); + (void)iter_getnext_op->set_attr_output_shapes(param.shapes()); + (void)iter_getnext_op->set_attr_channel_name(param.queue_name()); + + // save iter_getnext_op for later use + dataset_iter_getnext_ = iter_getnext_op; + } + + // return the data flow graph + return *this; +} + +void DfGraphConvertor::TraceOutputFromTupleGetItem(const AnfNodePtr &anf_out) { + auto it = out_handle_cache_.find(anf_out.get()); + if (it != out_handle_cache_.end()) { + OutHandler handle = it->second; + auto op = handle.op; + if (op != nullptr) { + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType() << ", out_name: " << handle.out; + (void)graph_outputs_.emplace_back(*op, handle.out); + } else { + MS_LOG(EXCEPTION) << "tuple_getitem: " << anf_out->fullname_with_scope() << " is not converted"; + } + } else { + // invalid tuple_getitem e.g. tuple_getitem(tuple_getitem())/tuple_getitem(depend())/tuple_getitem(make_tuple()) + MS_LOG(WARNING) << "Invalid tuple_getitem: " << anf_out->fullname_with_scope(); + } +} + +void DfGraphConvertor::TraceOutput(const AnfNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + AnfNodePtr anf_out = node; + AnfNodePtr pre_node = nullptr; + + // Trace value node + if (node->isa()) { + auto op = Convert(anf_out); + if (op != nullptr) { + (void)graph_outputs_.emplace_back(*op, ""); + AddGraphConstInput(op); + } + return; + } + + // Trace Parameter node + TraceOutputFromParameter(anf_out); + // Then trace cnode + if (!node->isa()) { + return; + } + + // trace tuple_getitem + while (anf_out->isa() && IsPrimitiveCNode(anf_out, prim::kPrimTupleGetItem)) { + pre_node = anf_out; + anf_out = anf_out->cast()->input(1); + } + // trace every element of make_tuple + auto c = anf_out->cast(); + std::string name = ""; + if (anf_out->isa()) { + name = GetCNodeTargetFuncName(c); + } + + if (name == "MakeTuple") { + for (unsigned int i = 1; i < c->inputs().size(); i++) { + TraceOutput(c->input(i)); + } + } else if (name == prim::kPrimDepend->name()) { + if (c->inputs().size() < 3) { // "Depend" primitive have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs is " << c->inputs().size() << ", which is less than 3"; + } + TraceOutput(c->input(1)); + } else if (name == prim::kTupleGetItem) { + TraceOutputFromTupleGetItem(anf_out); + } else { + // add outputs + auto op = Convert(anf_out); + std::string index; + if (op != nullptr) { + if ((pre_node != nullptr) && IsPrimitiveCNode(pre_node, prim::kPrimTupleGetItem)) { + auto item = out_handle_cache_.find(pre_node.get()); + if (item != out_handle_cache_.end()) { + index = item->second.out; + } else { + MS_LOG(WARNING) << "Can't get operator: " << anf_out->fullname_with_scope() << " 's output item"; + } + } + MS_LOG(INFO) << "Add graph output: " << anf_out->fullname_with_scope() << ":" << index; + (void)graph_outputs_.emplace_back(*op, index); + } + } +} + +void DfGraphConvertor::TraceOutputFromParameter(const AnfNodePtr &anf_out) { + MS_EXCEPTION_IF_NULL(anf_out); + if (anf_out->isa()) { + MS_LOG(INFO) << "Add graph output: " << anf_out->fullname_with_scope(); + auto it = out_handle_cache_.find(anf_out.get()); + if (it != out_handle_cache_.end()) { + // For dataset graph mode, input parameter is converted to a "iterator_get_next:yn" OutHandler. + OutHandler handle = it->second; + auto op = handle.op; + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType() << ", out_name: " << handle.out; + (void)graph_outputs_.emplace_back(*op, handle.out); + } else { + // common parameter case + auto op = Convert(anf_out); + if (op != nullptr) { + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType(); + (void)graph_outputs_.emplace_back(*op, ""); + } + } + } +} + +void SetupDatasetIterGetNextNode(const OperatorPtr &op) { + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + size_t output_num = param.ge_types().size(); + MS_LOG(INFO) << "Set iterator_getnext op's output num = " << output_num << "."; + // set iterator_getnext op's output num + shared_ptr iter_getnext = std::static_pointer_cast(op); + (void)iter_getnext->create_dynamic_output_y(static_cast(output_num)); + + for (uint32_t i = 0; i < output_num; i++) { + ge::TensorDesc desc(GeShape(param.shapes()[i]), ge::FORMAT_NCHW, (ge::DataType)param.ge_types()[i]); + // we don't SetRealDimCnt here since GE do not use this output's real-dim + (void)iter_getnext->update_dynamic_output_desc_y((i), desc); + } + } + return; +} + +void DfGraphConvertor::SetSubgraph(const AnfNodePtr &node) { + if (!node->isa()) { + return; + } + auto cnode = node->cast(); + if (!IsCaseNode(cnode)) { + return; + } + std::vector case_inputs; + for (size_t i = 1; i < cnode->inputs().size(); i++) { + case_inputs.emplace_back(cnode->input(i)); + } + std::shared_ptr> branches = std::make_shared>(); + auto bnode = cnode->input(0)->cast()->input(2)->cast(); + + for (size_t i = 1; i < bnode->inputs().size(); i++) { + auto branch_node = bnode->input(i)->cast(); + for (size_t j = 2; j < branch_node->inputs().size(); j++) { + if (std::find(case_inputs.begin(), case_inputs.end(), branch_node->input(j)) == case_inputs.end()) { + case_inputs.emplace_back(branch_node->input(j)); + } + } + } + + for (size_t i = 1; i < bnode->inputs().size(); i++) { + ProcessSubgraph(bnode->input(i), case_inputs); + } + + for (size_t i = 1; i < bnode->inputs().size(); i++) { + (void)branches->emplace_back(branches_map_[bnode->input(i).get()]); + } + + if (op_cache_.find(node.get()) == op_cache_.end()) { + return; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + MS_LOG(DEBUG) << "Not found adapter"; + return; + } + + OperatorPtr op = Convert(node); + (void)adpt->setSubgraph(op, 0, branches); + return; +} + +void DfGraphConvertor::GetCaseNodeInput(const CNodePtr node, const CNodePtr input_node) { + std::vector case_inputs; + for (size_t i = 1; i < node->inputs().size(); i++) { + case_inputs.emplace_back(node->input(i)); + } + auto bnode = input_node->input(2)->cast(); + MS_EXCEPTION_IF_NULL(bnode); + for (size_t i = 1; i < bnode->inputs().size(); i++) { + auto branch_node = bnode->input(i)->cast(); + MS_EXCEPTION_IF_NULL(branch_node); + for (size_t j = 2; j < branch_node->inputs().size(); j++) { + if (std::find(case_inputs.begin(), case_inputs.end(), branch_node->input(j)) == case_inputs.end()) { + case_inputs.emplace_back(branch_node->input(j)); + } + } + } + + const size_t case_index = 1; + const size_t make_tuple_index = 2; + + AnfNodePtr case_index_iter = input_node->input(case_index); + AnfNodePtr make_tuple_iter = input_node->input(make_tuple_index); + auto make_tuple_node = make_tuple_iter->cast(); + std::shared_ptr> tuple_items = std::make_shared>(); + + for (size_t i = 0; i < case_inputs.size(); i++) { + auto item = case_inputs[i]; + auto op = Convert(item); + if (op != nullptr) { + (void)tuple_items->emplace_back(OutHandler(op, "", item)); + } else if (out_handle_cache_.find(item.get()) != out_handle_cache_.end()) { + tuple_items->push_back(out_handle_cache_[item.get()]); + } else { + MS_LOG(DEBUG) << "Add an empty out handler: " << item->ToString(); + tuple_items->emplace_back(OutHandler()); + } + } + + tuple_out_handle_cache_[make_tuple_node.get()] = tuple_items; + + std::shared_ptr> case_input_items = std::make_shared>(); + (void)case_input_items->emplace_back(case_index_iter); + (void)case_input_items->emplace_back(make_tuple_iter); + case_input_handle_cache_[node.get()] = case_input_items; +} + +void DfGraphConvertor::UpdateTupleOutCache() { + for (auto &it : tuple_out_handle_cache_) { + std::size_t len = it.second->size(); + for (std::size_t i = 0; i < len; i++) { + OutHandler handle = (*it.second)[i]; + if (handle.op == nullptr) { + continue; + } + string name = handle.op->GetName(); + if (vars_.count(name) && (vars_[name] != nullptr)) { + (*it.second)[i] = OutHandler(vars_[name], handle.out, handle.node); + MS_LOG(INFO) << "update tuple_out_handle_cache_ " << name; + } + } + } +} + +DfGraphConvertor &DfGraphConvertor::BuildGraph() { + SetupDatasetIterGetNextNode(dataset_iter_getnext_); + + if (error_ != SUCCESS) { + return *this; + } + + // Case node set input. + std::vector nodes = GetOrderedCNodes(anf_graph_); + for (auto &it : nodes) { + if (it->isa() && IsCaseNode(it->cast())) { + auto node = it->cast(); + auto input_node = node->input(0)->cast(); + GetCaseNodeInput(node, input_node); + } + } + + // update tuple_out_handle_cache_ + UpdateTupleOutCache(); + + // set up dependencies + MS_LOG(DEBUG) << "set up dependencies"; + nodes = GetOrderedCNodes(anf_graph_); + for (auto &it : nodes) { + SetNodeInput(it); + SetOpControlInput(it); + SetSubgraph(it); + UpdateOpDesc(it); + } + + if (error_ == SUCCESS) { + df_graph_ = make_shared(anf_graph_->ToString()); + } else { + return *this; + } + + // set graph input according to the order from anf graph + std::vector inputs; + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + inputs.push_back(*dataset_iter_getnext_); + } else { + auto params = anf_graph_->parameters(); + if (use_inputs_) { + params = inputs_; + auto anf_params = anf_graph_->parameters(); + for (size_t i = 0; i < params.size(); i++) { + for (size_t j = 0; j < anf_params.size(); j++) { + if (params[i]->ToString() == anf_params[j]->ToString()) { + params[i] = anf_params[j]; + } + } + } + } + + int index = 0; + for (auto &it : params) { + auto name = std::static_pointer_cast(it)->name(); + // the parameters which has not been converted to var + if (vars_.find(name) == vars_.end()) { + if (HasAbstractMonad(it)) { + MS_LOG(INFO) << it->DebugString() << " is a monad parameter, skip."; + continue; + } + auto op = Convert(it); + MS_EXCEPTION_IF_NULL(op); + MS_LOG(INFO) << "add not var input " << it->ToString() << ", index " << index; + if (op == nullptr) { + MS_LOG(ERROR) << "Convert graph failed!"; + return *this; + } + UpdateDataOpDesc(it, op); + MS_LOG(INFO) << "add input " << it->ToString() << ", index " << index; + (void)std::static_pointer_cast(op)->set_attr_index(index++); + inputs.push_back(*op); + } else if (vars_[name] != nullptr) { + MS_LOG(INFO) << "add var input " << it->ToString(); + auto op = Convert(it); + UpdateConstOpDesc(it, vars_[name]); + MS_EXCEPTION_IF_NULL(op); + inputs.push_back(*op); + } + } + } + + MS_LOG(DEBUG) << "trace output"; + graph_outputs_.clear(); + TraceOutput(anf_graph_->get_return()->input(1)); + + // Add const nodes as graph input for some operator work with constant + MS_LOG(INFO) << "graph const input size: " << graph_const_inputs_.size(); + (void)std::transform(graph_const_inputs_.begin(), graph_const_inputs_.end(), std::back_inserter(inputs), + [](const OperatorPtr &x) { return *x; }); + + MS_LOG(INFO) << "set graph input num: " << inputs.size(); + (void)df_graph_->SetInputs(inputs); + + // set graph output + // set the value of finale return apply node as the output of dataflow graph + MS_LOG(DEBUG) << "set output"; + MS_LOG(INFO) << "set graph output num: " << graph_outputs_.size(); + (void)df_graph_->SetOutputs(graph_outputs_); + + compute_sout_ << "}" << endl; + // For the graph(e.g. eval_subgraph) whose IterNum is 1, donot set NeedIteration flag. + if (ConfigManager::GetInstance().iter_num() > 1) { + df_graph_->SetNeedIteration(true); + } + return *this; +} + +void DfGraphConvertor::UpdateConstOpDesc(const AnfNodePtr &it, const OperatorPtr &op) const { + if (!it->isa()) { + MS_LOG(DEBUG) << "It is not parameter, name: " << it->DebugString(); + return; + } + auto para = it->cast(); + MS_EXCEPTION_IF_NULL(para); + std::string format = kOpFormat_NCHW; + std::string param_debug_info = para->DebugString(); + auto param_format = param_format_.find(param_debug_info); + if (param_format != param_format_.end()) { + format = param_format->second; + MS_LOG(DEBUG) << "Parameter debug info: " << param_debug_info << ", format is " << format; + } + if (format == kOpFormat_NCHW) { + MS_LOG(DEBUG) << "Format is not changed, no need to update op desc, name: " << param_debug_info; + return; + } + if (!para->has_default()) { + MS_LOG(DEBUG) << "Parameter has no default, no need to update op desc, name: " << param_debug_info; + return; + } + auto value = para->default_param(); + MS_EXCEPTION_IF_NULL(value); + auto tensor = value->cast>(); + MS_EXCEPTION_IF_NULL(tensor); + auto const_op_desc = TransformUtil::GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + if (const_op_desc == nullptr) { + MS_LOG(WARNING) << "Create parameter " << para->name() << " output descriptor failed!"; + return; + } + (void)std::static_pointer_cast(op)->update_output_desc_y(*const_op_desc); +} + +void DfGraphConvertor::UpdateDataOpDesc(const AnfNodePtr &it, const OperatorPtr &op) const { + auto node = std::static_pointer_cast(it); + if (node == nullptr) { + MS_LOG(ERROR) << "Update data op descriptor failed! Invalid node."; + return; + } + + std::vector shape; + if (auto normal_shape_ptr = dyn_cast(node->Shape()); normal_shape_ptr != nullptr) { + shape = normal_shape_ptr->shape(); + } else if (auto no_shape_ptr = dyn_cast(node->Shape()); no_shape_ptr != nullptr) { + shape = {}; + } else { + MS_LOG(INFO) << "Invalid shape to update data op descriptor."; + return; + } + + if (node->Type() == nullptr) { + MS_LOG(INFO) << "Invalid type to update data op descriptor."; + return; + } + TypeId me_type = node->Type()->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(node->Type())->element()->type_id(); + } + std::ostringstream buf; + buf << "[" << shape << "]"; + MS_LOG(INFO) << "input shape is " << buf.str() << ", type is " << me_type; + std::string format = "NCHW"; + if (it->isa()) { + auto param = it->cast(); + std::string param_name = param->DebugString(); + auto param_format = param_format_.find(param_name); + if (param_format != param_format_.end()) { + format = param_format->second; + MS_LOG(DEBUG) << "parameter: " << param_name << ", format is " << format; + } + } + auto desc = TransformUtil::GetGeTensorDesc(shape, me_type, format); + if (desc == nullptr) { + MS_LOG(ERROR) << "Update data op descriptor failed! TensorDesc is null."; + } else { + (void)std::static_pointer_cast(op)->update_input_desc_x(*desc); + (void)std::static_pointer_cast(op)->update_output_desc_y(*desc); + } +} + +DfGraphPtr DfGraphConvertor::GetComputeGraph() { return df_graph_; } + +DfGraphPtr DfGraphConvertor::GetInitGraph() { return init_graph_; } + +DfGraphPtr DfGraphConvertor::GetSaveCheckpointGraph() { return save_ckp_graph_; } + +DfGraphPtr DfGraphConvertor::GetBroadcastGraph() { return broadcast_graph_; } + +bool DfGraphConvertor::IsSourceEdgeNode(const AnfNodePtr &node) { + if (!node->isa()) { + return false; + } + auto cnode = node->cast(); + if (!IsCustomCNode(cnode)) { + std::string name = GetCNodeTargetFuncName(cnode); + if (name.empty()) { + return false; + } + + // Ignore apply node Depend, UpdateState, make_tuple. make_tuple in ge pipeline. + if ((name == prim::kPrimDepend->name()) || (name == prim::kPrimUpdateState->name()) || + (name == prim::kPrimReturn->name()) || (name == prim::kPrimMakeTuple->name())) { + return false; + } + } + // Load and other normal primitives which contain monad node. + auto has_monad = std::any_of(cnode->inputs().begin(), cnode->inputs().end(), + [](const AnfNodePtr &node) -> bool { return HasAbstractMonad(node); }); + if (has_monad) { + return true; + } + + // primitive with make_tuple as input + for (auto &input : cnode->inputs()) { + if (IsPrimitiveCNode(input, prim::kPrimMakeTuple)) { + auto tuple = input->cast(); + auto ret = std::any_of(tuple->inputs().begin(), tuple->inputs().end(), + [](const AnfNodePtr &node) -> bool { return HasAbstractMonad(node); }); + if (ret) { + return true; + } + } + } + + return false; +} + +bool DfGraphConvertor::IsControlEdgeNode(const AnfNodePtr &node) { + if (!node->isa()) { + return false; + } + auto cnode = node->cast(); + if (!IsCustomCNode(cnode)) { + std::string name = GetCNodeTargetFuncName(cnode); + if (name.empty()) { + return false; + } + + // Ignore apply node of Load, Depend, UpdateState, make_tuple, return + if ((name == prim::kPrimLoad->name()) || (name == prim::kPrimDepend->name()) || + (name == prim::kPrimUpdateState->name()) || (name == prim::kPrimMakeTuple->name()) || + (name == prim::kPrimReturn->name())) { + return false; + } + } + return true; +} + +OperatorPtr DfGraphConvertor::ToOperatorPtr(const AnfNodePtr &node) { + auto op = Convert(GetRealOpNode(node)); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert real op node to operator failed, " << node->ToString(); + error_ = FAILED; + return nullptr; + } + return op; +} + +void DfGraphConvertor::AddEdgeToCache(const AnfNodePtr &src, const AnfNodePtr &dest) { + auto item = monad_control_edge_cache_.find(src); + if (item == monad_control_edge_cache_.end()) { + monad_control_edge_cache_[src] = std::set{dest}; + } else { + (void)item->second.insert(dest); + } +} + +void DfGraphConvertor::AddEdgeForLoad(const AnfNodePtr &node) { + auto func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + auto mng = func_graph->manager(); + if (mng == nullptr) { + mng = Manage(func_graph, true); + func_graph->set_manager(mng); + } + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(node) == manager->node_users().end()) { + MS_LOG(EXCEPTION) << "Can't find node in nodes_users."; + } + auto &users = manager->node_users()[node]; + std::shared_ptr> src_node_list = std::make_shared>(); + std::shared_ptr> dst_node_list = std::make_shared>(); + for (const auto &iter : users) { + auto user_node = iter.first; + auto name = GetCNodeTargetFuncName(user_node->cast()); + if (name == prim::kPrimUpdateState->name()) { + FindDestOps(user_node, dst_node_list, false); + continue; + } + if (IsControlEdgeNode(user_node)) { + src_node_list->push_back(user_node); + continue; + } + FindDestOps(user_node, src_node_list, false); + } + + // add to cache + for (auto &dest : *dst_node_list) { + for (auto &src : *src_node_list) { + AddEdgeToCache(src, dest); + } + } +} + +void DfGraphConvertor::FindDestOps(const AnfNodePtr &node, const std::shared_ptr> &node_list, + bool top) { + MS_EXCEPTION_IF_NULL(node); + auto func_graph = node->func_graph(); + MS_EXCEPTION_IF_NULL(func_graph); + auto mng = func_graph->manager(); + if (mng == nullptr) { + mng = Manage(func_graph, true); + func_graph->set_manager(mng); + } + auto manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + + auto users = manager->node_users()[node]; + for (const auto &iter : users) { + auto user_node = iter.first; + if (IsControlEdgeNode(user_node)) { + if (!top) { + node_list->push_back(user_node); + } + } else { + FindDestOps(user_node, node_list, false); + } + } +} + +void DfGraphConvertor::AutoMonadCollectInput(const AnfNodePtr &node) { + if (!IsSourceEdgeNode(node)) { + return; + } + + // Add control edge if contain monad input. + std::string name = GetCNodeTargetFuncName(node->cast()); + if (name == prim::kPrimLoad->name()) { + AddEdgeForLoad(node); + } else { + auto src_ops = ToOperatorPtr(node); + if (src_ops != nullptr) { + // Find dest ops list + std::shared_ptr> dst_node_list = std::make_shared>(); + FindDestOps(node, dst_node_list, true); + for (auto &dest : *dst_node_list) { + AddEdgeToCache(node, dest); + } + } + } +} + +void DfGraphConvertor::AutoMonadSetInput(const AnfNodePtr &node) { + if (monad_control_edge_cache_.find(node) == monad_control_edge_cache_.end()) { + return; + } + + auto src_ops = ToOperatorPtr(node); + if (src_ops != nullptr) { + for (auto &dest : monad_control_edge_cache_[node]) { + auto dest_ops = ToOperatorPtr(dest); + if (dest_ops == nullptr) { + continue; + } + (void)dest_ops->AddControlInput(*src_ops); +#ifdef DRAW_GE_GRAPH + compute_sout_ << op_draw_name_[node.get()] << " -> " << op_draw_name_[dest.get()] << "[style=\"dotted\"]" << endl; +#endif + } + } +} + +void DfGraphConvertor::AutoMonadSetControlInput(const AnfNodePtr &node) { + AutoMonadCollectInput(node); + AutoMonadSetInput(node); +} + +void DfGraphConvertor::SetOpControlInput(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); + AutoMonadSetControlInput(node); + if (control_edge_cache_.find(node.get()) == control_edge_cache_.end()) { + return; + } + + std::vector control_edges = control_edge_cache_[node.get()]; + if ((control_edges.empty())) { + MS_LOG(ERROR) << "Get control edge node's src or dest operator failed"; + return; + } + + for (auto &item : control_edges) { + (void)item.dest_op->AddControlInput(*item.src_op); + } +} + +const std::vector trans_var_list = {string(kNameAssign), string(kNameAssignAdd), string(kNameAssignSub)}; + +AnfNodePtr DfGraphConvertor::ParseLoadInput(const CNodePtr &cnode) { + if (cnode->inputs().size() < 3) { + MS_LOG(EXCEPTION) << "input size error, " << cnode->ToString(); + } + const size_t para_index = 1; + return cnode->input(para_index); +} + +void DfGraphConvertor::SetTupleOpInput(const OpAdapterPtr &adpt, const CNodePtr &node, const AnfNodePtr &pred, + const OperatorPtr &src, int index) { + std::shared_ptr> handler_vec = tuple_out_handle_cache_[pred.get()]; + std::shared_ptr> handler_vec_without_monad = std::make_shared>(); + bool with_monad = false; + for (auto &handler : *handler_vec) { + // when tuple with monad type element, the handler operator is nullptr, should be ignored. + if (handler.op == nullptr) { + if ((handler.node != nullptr) && !HasAbstractMonad(handler.node)) { + MS_LOG(WARNING) << "Unsupported node in tuple : " << node->ToString(); + } + continue; + } + with_monad = true; + handler_vec_without_monad->push_back(handler); + } + int ret = adpt->setInput(src, index, handler_vec_without_monad); + if ((ret == 0) && pred->isa() && (pred->cast()->inputs().size() == handler_vec->size() + 1)) { + for (unsigned int j = 0; j < handler_vec_without_monad->size(); j++) { + AnfNodePtr input_node = pred->cast()->input(j + 1); + if (with_monad) { + input_node = handler_vec_without_monad->at(j).node; + } + compute_sout_ << op_draw_name_[input_node.get()] << " -> " << op_draw_name_[node.get()] << ":" << index << endl; + AddGraphConstInput(handler_vec_without_monad->at(j).op); + } + return; + } + MS_LOG(WARNING) << "This anf node is not supported as a tuple item : " << node->ToString(); +} +AnfNodePtr DfGraphConvertor::GetRealInputNode(const CNodePtr &node, const AnfNodePtr &input) { + if (input == nullptr || node == nullptr) { + return nullptr; + } + AnfNodePtr pred = input; + while (pred->isa() && GetCNodeTargetFuncName(pred->cast()) == prim::kPrimDepend->name()) { + pred = pred->cast()->input(1); + } + + // skip input of UMonad, IOMonad + if (IsValueNode(pred) || IsValueNode(pred)) { + return nullptr; + } + + // skip input of the None, UpdateState + if (IsValueNode(pred) || IsPrimitiveCNode(pred, prim::kPrimUpdateState)) { + return nullptr; + } + + if (IsPrimitiveCNode(pred, prim::kPrimLoad)) { + pred = ParseLoadInput(pred->cast()); + } + + // transform "Const" op to "Variable" op when the next node is "Assign" op. + std::string c_name = GetCNodeTargetFuncName(node); + auto pos = std::find(trans_var_list.begin(), trans_var_list.end(), c_name); + if (!training_ && pos != trans_var_list.end() && pred->isa()) { + std::string name = std::static_pointer_cast(pred)->name(); + auto op_itor = op_cache_.find(pred.get()); + if (op_itor == op_cache_.end()) { + MS_LOG(EXCEPTION) << "Can not find op for node " << pred->ToString() << "."; + } + if (op_itor->second != nullptr && + (op_itor->second->GetOpType() == "Constant" || op_itor->second->GetOpType() == "Const") && + vars_.find(name) != vars_.end()) { + auto variable = std::make_shared(name); + auto desc = vars_[name]->GetOutputDesc("y"); + (void)variable->update_output_desc_y(desc); + MS_LOG(DEBUG) << "Trans to variable, var = " << variable->GetName() << "."; + op_itor->second = variable; // replace parameter with variable + vars_[name] = variable; + } + } + return pred; +} + +void DfGraphConvertor::SetOpInput(const OpAdapterPtr &adpt, const CNodePtr &node) { + OperatorPtr src = Convert(node); + int case_flag = 0; + auto &inputs = node->inputs(); + size_t input_size = inputs.size(); + if (case_input_handle_cache_.find(node.get()) != case_input_handle_cache_.end()) { + case_flag = 1; + input_size = case_input_handle_cache_[node.get()]->size() + 1; + } + + for (size_t i = 1; i < input_size; i++) { + AnfNodePtr pred = nullptr; + if (case_flag != 0) { + pred = case_input_handle_cache_[node.get()]->at(i - 1); + } else { + pred = inputs[i]; + } + pred = GetRealInputNode(node, pred); + if (pred == nullptr) { + continue; + } + + int index = SizeToInt(i); + // find in out_hadnle_cache_ first + auto it = out_handle_cache_.find(pred.get()); + if (it != out_handle_cache_.end()) { + int ret = adpt->setInput(src, index, it->second); + if (ret == 0) { + if (pred->isa() && GetCNodeTargetFuncName(pred->cast()) == prim::kTupleGetItem) { + compute_sout_ << op_draw_name_[pred->cast()->input(1).get()] << " -> " << op_draw_name_[node.get()] + << ":" << i << endl; + } else if (pred->isa()) { + compute_sout_ << op_draw_name_[pred.get()] << " -> " << op_draw_name_[node.get()] << ":" << i << endl; + } else { + // don't draw anything. + MS_LOG(INFO) << "DRAW_GE_GRAPH: Shouldn't have this case."; + } + AddGraphConstInput(it->second.op); + } + } else if (tuple_out_handle_cache_.find(pred.get()) != tuple_out_handle_cache_.end()) { + SetTupleOpInput(adpt, node, pred, src, index); + } else { + auto op = Convert(pred); + int ret = adpt->setInput(src, index, op); + if (ret == 0) { + compute_sout_ << op_draw_name_[pred.get()] << " -> " << op_draw_name_[node.get()] << ":" << i << endl; + AddGraphConstInput(op); + } + } + } +} + +void DfGraphConvertor::AddGraphConstInput(const OperatorPtr &op) { + if (op->GetOpType() == "Constant" || op->GetOpType() == "Const") { + graph_const_inputs_.push_back(op); + } +} + +void DfGraphConvertor::SetNodeInput(const AnfNodePtr node) { + if (!node->isa()) { + return; + } + if (op_cache_.find(node.get()) == op_cache_.end()) { + return; + } + auto cnode = node->cast(); + OpAdapterPtr adpt = FindAdapter(cnode, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return; + } + + // get Operator from op_cache_, use adapter to set Inputs + DfGraphConvertor::SetOpInput(adpt, cnode); +} + +void DfGraphConvertor::ProcessSubgraph(const AnfNodePtr &node, const std::vector &inputs) { + if (!node->isa() || GetCNodeFuncName(node->cast()) != "Partial") { + return; + } + auto graph_node = node->cast()->input(1)->cast(); + MS_EXCEPTION_IF_NULL(graph_node); + FuncGraphPtr anf_graph = graph_node->value()->cast(); + DfGraphConvertor converter(anf_graph); + converter.use_inputs_ = true; + converter.inputs_ = inputs; + (void)converter.ConvertAllNode().BuildGraph(); +#ifdef ENABLE_DUMP_IR + std::string name = graph_node->ToString() + "_ge_graph.dot"; + if (MsContext::GetInstance()->get_param(MS_CTX_SAVE_GRAPHS_FLAG)) { + converter.DrawComputeGraph(name); + } +#endif + branches_map_[node.get()] = *(converter.df_graph_); +} + +// Update GE op's shape and type info +void DfGraphConvertor::UpdateOpDesc(const AnfNodePtr node) { + if (node == nullptr || !node->isa()) { + return; + } + + if (op_cache_.find(node.get()) == op_cache_.end()) { + return; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return; + } + + // get Operator from op_cache_ + OperatorPtr op = Convert(node); + + adpt->updateOutputDesc(op, node->Shape(), node->Type(), node); +} + +OperatorPtr DfGraphConvertor::Convert(const AnfNodePtr node) { + if (node == nullptr) { + MS_LOG(ERROR) << "node is nullptr"; + error_ = NOT_FOUND; + return nullptr; + } + // find in cache + if (op_cache_.count(node.get())) { + return op_cache_[node.get()]; + } + + // do not convert primitive node, Load, UpdateState + if (IsValueNode(node) || IsPrimitiveCNode(node, prim::kPrimLoad) || + IsPrimitiveCNode(node, prim::kPrimUpdateState)) { + return nullptr; + } + + // convert a new one + if (node->isa()) { + return ConvertCNode(node->cast()); + } + if (node->isa()) { + return ConvertParameter(node); + } + if (node->isa()) { + if (IsValueNode(node)) { + return nullptr; + } + return ConvertValueNode(node->cast()); + } + + MS_LOG(ERROR) << "Invalid AnfNode"; + error_ = INVALID_ARGUMENT; + return nullptr; +} + +void DfGraphConvertor::ConvertMakeTuple(const CNodePtr node) { + std::shared_ptr> tuple_items = std::make_shared>(); + // convert each tuple item to a OutHandler + for (size_t i = 1; i < node->inputs().size(); i++) { + AnfNodePtr item = node->input(i); + if (IsPrimitiveCNode(item, prim::kPrimLoad)) { + item = ParseLoadInput(item->cast()); + } + OperatorPtr op = Convert(item); + if (op != nullptr) { + (void)tuple_items->emplace_back(OutHandler(op, "", item)); + } else if (out_handle_cache_.find(item.get()) != out_handle_cache_.end()) { + tuple_items->push_back(out_handle_cache_[item.get()]); + } else { + tuple_items->emplace_back(OutHandler(nullptr, "", item)); + } + } + + MS_LOG(DEBUG) << "ConvertMakeTuple: " << node.get() << " " << tuple_items->size(); + tuple_out_handle_cache_[node.get()] = tuple_items; +} + +void DfGraphConvertor::ConvertTopK(const CNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + MS_LOG(INFO) << "Convert TopK second input's type from int64 to int32."; + auto value_ptr = node->input(2)->cast(); + MS_EXCEPTION_IF_NULL(value_ptr); + std::ostringstream ss; + ss << "op" << value_ptr.get(); + op_draw_name_[value_ptr.get()] = ss.str(); + compute_sout_ << ss.str() << "[label= \"" << value_ptr->value()->ToString() << "\" shape=ellipse]" << endl; + auto input_value = value_ptr->value(); + auto int64_value = GetValue(input_value); + OpAdapterPtr adpt = FindAdapter(value_ptr, training_); + auto op = adpt->generate(value_ptr); + (void)adpt->setAttr(op, "value", static_cast(int64_value)); + op_cache_[value_ptr.get()] = op; +} + +std::vector DfGraphConvertor::CastToInt(const ValuePtr &value) { + if (value == nullptr) { + MS_LOG(WARNING) << "Value ptr is nullptr."; + return {}; + } + std::vector cur_value = {}; + if (utils::isa(value)) { + auto val_seq_ptr = value->cast(); + MS_EXCEPTION_IF_NULL(val_seq_ptr); + if (!val_seq_ptr->value().empty()) { + auto first_val = val_seq_ptr->value().front(); + MS_EXCEPTION_IF_NULL(first_val); + MS_EXCEPTION_IF_NULL(first_val->type()); + if (first_val->type()->number_type() == kNumberTypeInt64) { + cur_value = GetValue>(value); + } else { + auto origin_value = GetValue>(value); + (void)std::transform(origin_value.begin(), origin_value.end(), std::back_inserter(cur_value), + [](int index) { return static_cast(index); }); + } + } + } else { + MS_EXCEPTION_IF_NULL(value->type()); + if (value->type()->number_type() == kNumberTypeInt64) { + cur_value.push_back(GetValue(value)); + } else { + cur_value.push_back(static_cast(GetValue(value))); + } + } + return cur_value; +} + +void DfGraphConvertor::ConvertReshape(const CNodePtr node) { + MS_LOG(INFO) << "Convert the second input of reshape to op attr."; + const auto kInputNum = 3; + if (node->size() < kInputNum) { + MS_LOG(WARNING) << "Reshape must have two inputs."; + return; + } + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + return; + } + auto op = adpt->generate(node); + MS_EXCEPTION_IF_NULL(op); + // get shape form attr + auto value_node = node->input(0)->cast(); + MS_EXCEPTION_IF_NULL(value_node); + MS_EXCEPTION_IF_NULL(value_node->value()); + auto primitive = value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(primitive); + auto value = primitive->GetAttr("shape"); + std::vector list; + list = CastToInt(value); + + (void)op->SetAttr("shape", list); + op_cache_[node.get()] = op; +} + +void DfGraphConvertor::ConvertConv2D(const CNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + return; + } + auto op = adpt->generate(node); + MS_EXCEPTION_IF_NULL(op); + auto value_node = node->input(0)->cast(); + MS_EXCEPTION_IF_NULL(value_node); + MS_EXCEPTION_IF_NULL(value_node->value()); + auto primitive = value_node->value()->cast(); + MS_EXCEPTION_IF_NULL(primitive); + auto value = primitive->GetAttr("padding"); + if (value != nullptr) { + std::string pad_mode = GetValue(value); + (void)op->SetAttr("padding", pad_mode); + } + op_cache_[node.get()] = op; +} + +AnfNodePtr DfGraphConvertor::TraceTupleGetItem(const CNodePtr &node, uint64_t *index) { + const int TUPLE_GET_ITEM_INDEX = 2; + if (node->inputs().size() < 3) { // "tuple_getitem" primitive must have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs of TupleGetItem is less than 3"; + } + auto index_node = node->inputs()[TUPLE_GET_ITEM_INDEX]; + if (!index_node->isa()) { + error_ = INVALID_ARGUMENT; + MS_LOG(EXCEPTION) << "can't convert get item with non-constant index"; + } + *index = LongToUlong(GetValue(GetValueNode(index_node))); + return node->inputs()[1]; +} + +AnfNodePtr DfGraphConvertor::TraceDepend(const CNodePtr &node) { + auto cnode = node->cast(); + if (cnode->inputs().size() < 3) { // "Depend" primitive have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs of depend is less than 3"; + } + return cnode->inputs()[1]; +} + +AnfNodePtr DfGraphConvertor::TraceMakeTuple(const CNodePtr &node, uint64_t index) { + if (index + 1 >= node->inputs().size()) { + MS_LOG(EXCEPTION) << "length of make_tuple is less than index: " << index; + } + return node->inputs()[index + 1]; +} + +OutHandler DfGraphConvertor::GetHandler(const AnfNodePtr &node, const std::stack &index_stack, + AnfNode *const draw_index) { + if (node == nullptr) { + MS_LOG(ERROR) << "Get nullptr while trace real op"; + return OutHandler(nullptr, ""); + } + std::ostringstream ss; + ss << "op" << node.get(); + if (index_stack.empty()) { + op_draw_name_[draw_index] = ss.str(); + return OutHandler(Convert(node), ""); + } else { + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + MS_LOG(ERROR) << "Can not get node output as adpt is nullptr!"; + error_ = NOT_FOUND; + return OutHandler(nullptr, ""); + } + OperatorPtr op = Convert(node); + if (op == nullptr) { + error_ = NOT_FOUND; + MS_LOG(ERROR) << "Can not convert node for trace real op"; + return OutHandler(nullptr, ""); + } + op_draw_name_[draw_index] = ss.str(); + return adpt->getOutput(Convert(node), static_cast(index_stack.top())); + } +} + +// get the real operator through maketuple tuple_getitem depend +OutHandler DfGraphConvertor::TraceRealOp(AnfNodePtr node) { + bool flag = IsPrimitiveCNode(node, prim::kPrimTupleGetItem) || IsPrimitiveCNode(node, prim::kPrimMakeTuple) || + IsPrimitiveCNode(node, prim::kPrimDepend); + std::stack index_stack; + auto draw_index = node.get(); + while (flag) { + flag = false; + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + uint64_t index; + node = TraceTupleGetItem(node->cast(), &index); + index_stack.push(index); + flag = true; + } else if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + if (index_stack.empty()) { + MS_LOG(ERROR) << "TraceRealOp find a make_tuple node"; + return OutHandler(nullptr, ""); + } else { + node = TraceMakeTuple(node->cast(), index_stack.top()); + index_stack.pop(); + flag = true; + } + } else if (IsPrimitiveCNode(node, prim::kPrimDepend)) { + node = TraceDepend(node->cast()); + flag = true; + } + } + return GetHandler(node, index_stack, draw_index); +} + +void DfGraphConvertor::ConvertTupleGetItem(const CNodePtr node) { + auto handle = TraceRealOp(node); + if (handle.op == nullptr) { + MS_LOG(ERROR) << "Failed to trace tuple get item"; + return; + } + out_handle_cache_[node.get()] = handle; +} + +// Get the real op for tuple_getitem through make tuple, or depend +AnfNodePtr DfGraphConvertor::GetRealOpNode(AnfNodePtr node) { + const int TUPLE_GET_ITEM_INDEX = 2; + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + auto node_inputs = node->cast()->inputs(); + if (node_inputs.size() != 3) { // "tuple_getitem" primitive must have 3 inputs + MS_LOG(ERROR) << "tuple get item node not correct!"; + error_ = FAILED; + return node; + } + MS_EXCEPTION_IF_NULL(node_inputs[TUPLE_GET_ITEM_INDEX]); + if (!node_inputs[TUPLE_GET_ITEM_INDEX]->isa()) { + error_ = INVALID_ARGUMENT; + MS_LOG(EXCEPTION) << "can't convert get item with non-constant index"; + } + auto value_ptr = GetValueNode(node_inputs[TUPLE_GET_ITEM_INDEX])->cast(); + if (value_ptr == nullptr) { + MS_LOG(ERROR) << "Can not convert get item as value is nullptr!"; + error_ = FAILED; + return node; + } + int64_t index = value_ptr->value(); + + // make_tuple apply inputs:make_tuple, [tuple_items,] + if (IsPrimitiveCNode(node_inputs[1], prim::kPrimMakeTuple)) { + auto tuple_inputs = node->cast()->inputs(); + if (tuple_inputs.size() < LongToSize(index + 1L)) { + MS_LOG(ERROR) << "make tuple input items node not correct! size:" << tuple_inputs.size() + << ", item index:" << index; + error_ = FAILED; + return node; + } + return GetRealOpNode(tuple_inputs[LongToSize(index + 1L)]); + } + return GetRealOpNode(node_inputs[1]); + } + + // depend apply inputs: depend,output,depended_node + if (IsPrimitiveCNode(node, prim::kPrimDepend)) { + auto depend_inputs = node->cast()->inputs(); + if (depend_inputs.size() != 3) { // "Depend" primitive have 3 inputs + MS_LOG(ERROR) << "depend input items not correct"; + error_ = FAILED; + return node; + } + return GetRealOpNode(depend_inputs[1]); + } + return node; +} + +// convert the anf node to corresponding operator list +std::vector DfGraphConvertor::ConvertDependNode(const AnfNodePtr node) { + if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + std::vector op_lists; + auto node_inputs = node->cast()->inputs(); + for (size_t index = 1; index < node_inputs.size(); index++) { + auto op = Convert(GetRealOpNode(node_inputs[index])); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert real op node to operator failed"; + error_ = FAILED; + return std::vector({}); + } + op_lists.push_back(op); + } + return op_lists; + } + + auto op = Convert(GetRealOpNode(node)); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert real op node to operator failed"; + error_ = FAILED; + return std::vector({}); + } + return std::vector({op}); +} + +bool DfGraphConvertor::CheckCNode(const std::string &name, const CNodePtr node) { + // ignore apply node of return + if (name == "" || name == prim::kPrimReturn->name() || name == prim::kPrimDepend->name() || + name == prim::kPrimSwitchLayer->name() || name == prim::kPrimPartial->name()) { + return false; + } + + // Convert TopK second input from int64 to int32. + if (name == prim::kPrimTopK->name()) { + ConvertTopK(node); + return true; + } + + // Convert Reshape add const input to attr(shape) + if (name == prim::kPrimReshape->name()) { + ConvertReshape(node); + return true; + } + + // Add attr pad mode to Conv2D + if (name == prim::kPrimConv2D->name() || name == prim::kPrimDepthwiseConv2dNative->name() || + name == kNameConv2DBackpropInputV2) { + ConvertConv2D(node); + return true; + } + + // make_tuple is used for a dynamic_input, convert it to a vector of OutHandlers + if (name == prim::kPrimMakeTuple->name()) { + ConvertMakeTuple(node); + return false; + } + + // As for nodes with multi outputs, convert tuple_getitem to OutHandle + if (name == prim::kPrimTupleGetItem->name()) { + ConvertTupleGetItem(node); + return false; + } + + return true; +} + +OperatorPtr DfGraphConvertor::ConvertCNode(const CNodePtr node) { + SaveParamFormat(node); + std::string name = GetCNodeTargetFuncName(node); + if (!CheckCNode(name, node)) { + return nullptr; + } + + // get corresponding OpAdapter + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return nullptr; + } + + // get operator + OperatorPtr op = nullptr; + auto it_op = op_cache_.find(node.get()); + if (it_op != op_cache_.end()) { + op = it_op->second; + } else { + op = adpt->generate(node); + } + + // set attribute for primitive + (void)adpt->setAttr(op, node); + + // add into cache + (void)op_cache_.emplace(node.get(), op); + + DrawCNode(node, adpt); + + return op_cache_[node.get()]; +} + +OperatorPtr DfGraphConvertor::ConvertParameter(const AnfNodePtr node) { + // convert Parameter in ANF to variable in DataFlow + auto adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + MS_LOG(EXCEPTION) << "Can not find adapter for Parameter"; + } + auto op = adpt->generate(node); + op_cache_[node.get()] = op; + + // build index for parameter using name + std::string name = std::static_pointer_cast(node)->name(); + params_[name] = node; + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + compute_sout_ << ss.str() << "[shape=octagon, label=\"" << name << "\"]" << endl; + return op_cache_[node.get()]; +} + +void DfGraphConvertor::SaveParamFormat(const CNodePtr node) { + AnfNodePtr op = node->input(0); + if (IsValueNode(op)) { + auto prim = GetValueNode(op); + for (auto attr : prim->attrs()) { + if (attr.first == "format") { + std::string format; + if (attr.second->isa()) { + bool converted = CheckAndConvertUtils::ConvertAttrValueToString(prim->name(), "format", &attr.second); + if (converted) { + format = attr.second->ToString(); + } else { + CheckAndConvertUtils::GetFormatStringVal(prim, &format); + } + } else if (attr.second->isa()) { + format = attr.second->ToString(); + } + if (format != "NCDHW" && format != "NHWC") { + break; + } + for (size_t i = 1; i < node->size(); i++) { + auto input = node->input(i); + if (input->isa()) { + param_format_[input->DebugString()] = format; + MS_LOG(DEBUG) << "Save Param " << input->DebugString() << " format: " << format; + } + } + } + } + } +} + +Status DfGraphConvertor::TryConvertValueNodeToMultiConst(const ValueNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + ValuePtr value = node->value(); + MS_EXCEPTION_IF_NULL(value); + if (!value->isa() && !value->isa()) { + return FAILED; + } + + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + if (vec.empty()) { + return FAILED; + } + + std::shared_ptr> tuple_items = std::make_shared>(); + for (size_t i = 0; i < vec.size(); i++) { + MS_EXCEPTION_IF_NULL(vec[i]); + if (vec[i]->isa()) { + GeTensorPtr ge_tensor = transform::TransformUtil::ConvertTensor(vec[i]->cast(), kOpFormat_NCHW); + auto const_op = std::make_shared(node->fullname_with_scope() + "/const/inputs/" + std::to_string(i)); + (void)const_op->set_attr_value(*ge_tensor); + (void)const_op->update_output_desc_y(ge_tensor->GetTensorDesc()); + (void)tuple_items->emplace_back(OutHandler(const_op, "")); + } else { + return FAILED; + } + } + if (tuple_items->empty()) { + return FAILED; + } + + tuple_out_handle_cache_[node.get()] = tuple_items; + return SUCCESS; +} + +OperatorPtr DfGraphConvertor::ConvertValueNode(const ValueNodePtr node) { + // convert valuenode in ANF to Const in DataFlow + // find paramerte referenced by SymbolicKeyInstance of valuenode + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + compute_sout_ << ss.str() << "[label= \"" << node->value()->ToString() << "\" shape=ellipse]" << endl; + + if (TryConvertValueNodeToMultiConst(node) == SUCCESS) { + MS_LOG(INFO) << "Convert value node to multi Constant OP success"; + return nullptr; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return nullptr; + } + auto op = adpt->generate(node); + // set const's attrs + if (adpt->setAttr(op, "value", node->value()) != 0) { + MS_LOG(WARNING) << "set attr value for const failed"; + } + + auto const_op = std::static_pointer_cast(op); + if (const_op == nullptr) { + MS_LOG(ERROR) << "Get Constant operator failed"; + return nullptr; + } + auto ge_tensor = const_op->get_attr_value(); + auto ge_desc = ge_tensor.GetTensorDesc(); + (void)const_op->update_output_desc_y(ge_desc); + op_cache_[node.get()] = op; + return op_cache_[node.get()]; +} + +void DfGraphConvertor::DrawCNode(const CNodePtr node, const OpAdapterPtr adpt) { + if (adpt == nullptr || node == nullptr) { + MS_LOG(ERROR) << "Failed to draw apply node as adpt or node is nullptr!"; + return; + } + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + + compute_sout_ << ss.str() << "[label=<"; + compute_sout_ << "" << endl; + + auto input_map = adpt->getInputMap(); + auto dyn_input_map = adpt->getDynInputMap(); + if (input_map.size() + dyn_input_map.size() > 0) { + compute_sout_ << ""; + for (auto &it : input_map) { + compute_sout_ << ""; + } + for (auto &it : dyn_input_map) { + compute_sout_ << ""; + } + compute_sout_ << "" << endl; + } + + compute_sout_ << "" << endl; + + // print attrs' values + auto atts = adpt->GetAttrsFromDrawGraph(); + for (auto &it : atts) { + compute_sout_ << ""; + } + + adpt->clearAttrVect(); + + compute_sout_ << "
" << it.second.name << "" << it.second.name << "
\"" << node->ToString() + << ":" << GetCNodeTargetFuncName(node) << "\"
\"" << it + << "\"
> shape=plaintext]" << endl; +} +void DfGraphConvertor::RegisterAdapter(const std::string &name, OpAdapterPtr adpt) { + OpAdapterMap::get()[name] = std::make_shared(adpt); +} +void DfGraphConvertor::RegisterAdapter(const std::string &name, OpAdapterPtr train_adpt, OpAdapterPtr infer_adpt) { + OpAdapterMap::get()[name] = std::make_shared(train_adpt, infer_adpt); +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From c7ef57f747a3b7f3b54d41e0765391ca5eef6ee9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:27:37 +0800 Subject: [PATCH 096/160] ADD file via upload --- .../transform/graph_ir/df_graph_manager.cc | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/df_graph_manager.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/df_graph_manager.cc b/mindspore2022/ccsrc/transform/graph_ir/df_graph_manager.cc new file mode 100644 index 00000000000..248c7aaa021 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/df_graph_manager.cc @@ -0,0 +1,305 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/df_graph_manager.h" + +#include + +#ifndef ENABLE_LITE_ACL +#include "include/common/utils/python_adapter.h" +#include "pipeline/jit/pipeline.h" +#endif +#ifndef NO_DLIB +#include "tdt/tsd_client.h" +#endif + +namespace mindspore { +namespace transform { + +//在DfGraphWrapper区域中定义名为DfGraphWrapper的函数 +DfGraphWrapper::DfGraphWrapper(const std::string &name, const int &id, const DfGraphPtr &graph_ptr, + const OptionMap &options) + : name_(name), id_(id), graph_ptr_(graph_ptr), options_(options) {} + +//在DfGraphWrapper区域中定义名为DfGraphManager的函数 +//其功能主要为令将其中的三个变量“清空” +DfGraphManager::DfGraphManager() { + graph_id_ = 0; + graph_runner_ptr_ = nullptr; + sess_ptr_ = nullptr; +} + +//在DfGraphManager区域中定义名为~DfGraphManager的函数 +/* +分别执行 DeleteGraphRunner();三个函数 + DeleteGeSession(); + ClearGraph(); +其次判断ENABLE_LITE_ACL是否被定义，若已经被定义则执行set_python_env_flag函数的操作，并结束条件命令 +*/ +DfGraphManager::~DfGraphManager() { + // in python first destroy after atexit but in c++ destroy before atexit + //在python中，destory在atexit之后，而在c++中，destory在atexit之前 + DeleteGraphRunner(); + DeleteGeSession(); + ClearGraph(); +#ifndef ENABLE_LITE_ACL + python_adapter::set_python_env_flag(false); +#endif +} + +//在DfGraphManager区域中定义名为GetInstance的函数 +//返回instance +DfGraphManager &DfGraphManager::GetInstance() { + static DfGraphManager instance; + return instance; +} + +//在DfGraphWrapper区域中定义名为GenerateId的函数，返回值为int型 +/* +令graph_id_自增1，其次判断其大小是否小于等于0，若是，则令其等于1（检验DfGraphManager中的语句是否成功执行） +输出消息日志"Generate graph Id :[*此处为graph_id_]" +并返回graph_id_ +*/ +int DfGraphManager::GenerateId() { + graph_id_++; + if (graph_id_ <= 0) { + graph_id_ = 1; + } + MS_LOG(INFO) << "Generate graph Id : " << graph_id_; + return graph_id_; +} + +//在DfGraphManager区域中定义名为AddGraph的函数，返回值为自定义类型Status +/* +判断name是否为空，若为空则输出错误日志"The graph name is null, add graph failed"并返回INVALID_ARGUMENT +判断graph_ptr是否为空，若为空则输出消息日志"The new graph [*此处为name]'s pointer is null, add graph failed"并返回INVALID_ARGUMENT +定义id，wrap_ptr，ret，运用函数进行name, id, graph_ptr, options等数据的组合存储，建立新的shared_ptr +判断ret的第二位是否为false，若是，则代表其已经存在并将被覆写，输出相应警告日志并将其错位储存 +最后输出消息日志"Add graph "[*此处为name]" to GraphManager success!"并返回return Status::SUCCESS + */ +//本函数主要执行已有数据的组合重建成新的graph操作，并可检验操作是否成功 +Status DfGraphManager::AddGraph(const std::string &name, const DfGraphPtr &graph_ptr, const OptionMap &options) { + std::lock_guard lg(lock_); + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null, add graph failed"; + return Status::INVALID_ARGUMENT; + } + + if (graph_ptr == nullptr) { + MS_LOG(INFO) << "The new graph {" << name << "}'s pointer is null, add graph failed"; + return Status::INVALID_ARGUMENT; + } + + int id = GenerateId(); + DfGraphWrapperPtr wrap_ptr = std::make_shared(name, id, graph_ptr, options); + auto ret = graphs_.emplace(name, wrap_ptr); + if (ret.second == false) { + MS_LOG(WARNING) << "The graph name:{ " << name << " }is already exists! The old graph will be overwritten!!"; + ret.first->second = wrap_ptr; + } + MS_LOG(INFO) << "Add graph " << name << " to GraphManager success!"; + return Status::SUCCESS; +} + +//在DfGraphManager区域中定义名为GetAllGraphs的函数，返回值为自定义类型std::vector +/* +令it在从第一位开始在图表范围内循环，用ss和ret储存it储存的图表内容的不同位置的数据，并用"{}"作为ss的开头和结尾 +输出消息日志"Return graphs:[*此处为ss构成的图标]" +返回ret +*/ +std::vector DfGraphManager::GetAllGraphs() { + std::lock_guard lg(lock_); + std::vector ret; + std::stringstream ss; + ss << "{ "; + for (auto it = graphs_.begin(); it != graphs_.end(); ++it) { + ss << it->first << ", "; + ret.emplace_back(it->second); + } + ss << "}"; + MS_LOG(INFO) << "Return graphs: " << ss.str(); + return ret; +} + +//在DfGraphManager区域中定义名为GetSavedGraphs的函数，返回值为std::set +//功能为返回saved_graphs_ +std::set DfGraphManager::GetSavedGraphs() { return saved_graphs_; } + +//在DfGraphManager区域中定义名为AddSavedGraphs的函数 +//功能为将id插入到saved_graphs_中 +void DfGraphManager::AddSavedGraphs(const std::string &id) { saved_graphs_.insert(id); } + +//在DfGraphManager区域中定义名为GetGraphByName的函数，返回值为自定义类型DfGraphWrapperPtr +/* +判断name是否为空，若是则输出错误日志 "The graph name is null"并返回nullptr +定义it并用其寻找储存graph的name，若其在graph中找不到name，则输出消息日志"Can't found graph name:[*此处为name]"并返回nullptr +输出消息日志"Return graph:[*此处为name]" +返回it->second +*/ +DfGraphWrapperPtr DfGraphManager::GetGraphByName(const std::string &name) { + std::lock_guard lg(lock_); + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null"; + return nullptr; + } + + auto it = graphs_.find(name); + if (it == graphs_.end()) { + MS_LOG(INFO) << "Can't found graph name: " << name; + return nullptr; + } + MS_LOG(INFO) << "Return graph: " << name; + return it->second; +} + +//在DfGraphManager区域中定义名为ClearGraph的函数，且不抛出异常 +/* +清空graphs_和anf_graphs_并输出消息日志 "Remove all graphs in GraphManager" +*/ +void DfGraphManager::ClearGraph() noexcept { + std::lock_guard lg(lock_); + graphs_.clear(); + anf_graphs_.clear(); + MS_LOG(INFO) << "Remove all graphs in GraphManager"; +} + +//在DfGraphManager区域中定义名为SetAnfGraph的函数 +/* +获取name属性并储存在df_graph变量中 +判断其是否为空，若是则输出错误日志"Can't found graph name:[*此处为name]" +将anf_graph_ptr存入anf_graphs_的[df_graph->id_]位置 +*/ +void DfGraphManager::SetAnfGraph(const std::string &name, const AnfGraphPtr &anf_graph_ptr) { + DfGraphWrapperPtr df_graph = GetGraphByName(name); + if (df_graph == nullptr) { + MS_LOG(ERROR) << "Can't found graph name: " << name; + return; + } + std::lock_guard lg(lock_); + anf_graphs_[df_graph->id_] = anf_graph_ptr; +} + +//在DfGraphManager区域中定义名为GetAnfGraph的函数，返回值为自定义类型AnfGraphPtr +/* +定义iter寻找graph_id，若找不到则输出错误日志"Can't found anf graph, graph_id =[*此处为graph_id]"并返回nullptr +返回iter->second +*/ +AnfGraphPtr DfGraphManager::GetAnfGraph(uint32_t graph_id) { + std::lock_guard lg(lock_); + auto iter = anf_graphs_.find(graph_id); + if (iter == anf_graphs_.end()) { + MS_LOG(ERROR) << "Can't found anf graph, graph_id = " << graph_id; + return nullptr; + } + + return iter->second; +} + +//在DfGraphManager区域内定义名为EraseAnfGraph的函数 +//功能为清空anf_gtaphs_ +void DfGraphManager::EraseAnfGraph() { + std::lock_guard lg(lock_); + anf_graphs_.clear(); +} + +//在DfGraphManager区域内定义名为SetGeSession的函数 +/* +检测sess_ptr是否为空，若是则输出错误日志"You are adding a empty Ge Session" +检测sess_ptr_是否为空，若是则输出消息日志"Add a new Ge Session success" +否则输出消息日志"Add a new Ge Session success, the old Ge Session will be overwritten!!" +用sess_ptr_指针储存sess_ptr的地址 +*/ +void DfGraphManager::SetGeSession(const std::shared_ptr &sess_ptr) { + std::lock_guard lg(lock_); + if (sess_ptr == nullptr) { + MS_LOG(WARNING) << "You are adding a empty Ge Session"; + } + + if (sess_ptr_ == nullptr) { + MS_LOG(INFO) << "Add a new Ge Session success"; + } else { + MS_LOG(INFO) << "Add a new Ge Session success, the old Ge Session will be overwritten!!"; + } + sess_ptr_ = sess_ptr; +} + +//在DfGraphManager区域内定义名为GetGeSession的函数，返回值为自定义类型std::shared_ptr +//函数功能为返回sess_ptr_ +std::shared_ptr DfGraphManager::GetGeSession() { + std::lock_guard lg(lock_); + return sess_ptr_; +} + +//在DfGraphManager区域内定义名为DeleteGeSession的函数 +/* +检验sess_ptr_是否为空，若是则输出消息日志"Ge Session is not exist" +否则令sess_ptr_为空并清空saved_graphs_，输出消息日志"Delete Ge Session success" +*/ +void DfGraphManager::DeleteGeSession() noexcept { + std::lock_guard lg(lock_); + if (sess_ptr_ == nullptr) { + MS_LOG(INFO) << "Ge Session is not exist"; + } else { + sess_ptr_ = nullptr; + saved_graphs_.clear(); + MS_LOG(INFO) << "Delete Ge Session success"; + } +} + +//在DfGraphManager区域内定义名为SetGraphRunner的函数 +/* +检验graph_runner_ptr是否为空，若为空则输出警告日志"You are adding a empty GraphRunner" +检验graph_runner_ptr_是否为空，若是则输出消息日志"Add a new GraphRunner success" + 否则输出消息日志"Add a new GraphRunner success, the old GraphRunner will be overwritten!!" +用graph_runner_ptr_指针储存graph_runner_ptr的地址 +*/ +void DfGraphManager::SetGraphRunner(const std::shared_ptr &graph_runner_ptr) noexcept { + std::lock_guard lg(lock_); + if (graph_runner_ptr == nullptr) { + MS_LOG(WARNING) << "You are adding a empty GraphRunner"; + } + + if (graph_runner_ptr_ == nullptr) { + MS_LOG(INFO) << "Add a new GraphRunner success"; + } else { + MS_LOG(INFO) << "Add a new GraphRunner success, the old GraphRunner will be overwritten!!"; + } + graph_runner_ptr_ = graph_runner_ptr; +} + +//在DfGraphManager区域内定义名为GetGraphRunner的函数 +//功能为返回graph_runner_ptr_ +std::shared_ptr DfGraphManager::GetGraphRunner() { + std::lock_guard lg(lock_); + return graph_runner_ptr_; +} + +//在DfGraphManager区域内定义名为DeleteGraphRunner的函数 +/* +检测graph_runner_ptr_是否为空，若是则输出消息日志"GraphRunner is not exist" +否则令graph_runner_ptr_为空并输出消息日志"Delete GraphRunner success" +*/ +void DfGraphManager::DeleteGraphRunner() noexcept { + std::lock_guard lg(lock_); + if (graph_runner_ptr_ == nullptr) { + MS_LOG(INFO) << "GraphRunner is not exist"; + } else { + graph_runner_ptr_ = nullptr; + MS_LOG(INFO) << "Delete GraphRunner success"; + } +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From f2cbc1a69b2648dd15a99d26ef0377958a078364 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:27:52 +0800 Subject: [PATCH 097/160] ADD file via upload --- .../ccsrc/transform/graph_ir/graph_builder.cc | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/graph_builder.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/graph_builder.cc b/mindspore2022/ccsrc/transform/graph_ir/graph_builder.cc new file mode 100644 index 00000000000..9d37528b526 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/graph_builder.cc @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/graph_builder.h" + +#include + +#include "ops/math_ops.h" + +namespace mindspore { +namespace transform { +//定义名为BuildMDDatasetGraph的函数，返回值为自定义类型DfGraphPtr +//此函数为建立MD数据设置表格，返回为用指针引出的一系列数据 +/* +首先输出消息日志 "BuildMDDatasetGraph."，其次初始化数据并获取变量param的name属性 +再其次借助SetInputs函数和SetOutputs函数分别设置图表的输入和输出 +最后返回dataset_graph +*/ +DfGraphPtr BuildMDDatasetGraph(const DatasetGraphParam ¶m) { + MS_LOG(INFO) << "BuildMDDatasetGraph."; + + // InitData + auto d = ge::op::InitData("init_data_tmp").set_attr_channel_name(param.queue_name()); + + // set graph inputs & outputs + std::vector inputs{d}; + std::vector outputs{d}; + DfGraphPtr dataset_graph = std::make_shared("dataset"); + (void)dataset_graph->SetInputs(inputs); + (void)dataset_graph->SetOutputs(outputs); + + return dataset_graph; +} +//定义名为BuildDatasetGraph的函数，返回值为自定义类型Status +/* +首先定义Status类型变量ret，并将phase存入graph_name +其次输出消息日志"BuildDatasetGraph begin. phase is [*此处为phase]" + "param is[*此处打印param]." +再其次，先建立MD数据设置表格，再在此基础上执行对graph_name和dataset_graph的进一步操作并存入ret +判断上述操作是否成功，若不成功则输出错误日志"BuildDatasetGraph failed."，若成功则输出消息日志"BuildDatasetGraph end." +最后返回ret +*/ +Status BuildDatasetGraph(const DatasetGraphParam ¶m, const std::string &phase) { + Status ret; + std::string graph_name = phase; + + MS_LOG(INFO) << "BuildDatasetGraph begin. phase is " << phase; + MS_LOG(INFO) << "param is " << param.ToString() << "."; + + DfGraphPtr dataset_graph = BuildMDDatasetGraph(param); + ret = DfGraphManager::GetInstance().AddGraph(graph_name, dataset_graph); + if (ret != Status::SUCCESS) { + MS_LOG(ERROR) << "BuildDatasetGraph failed."; + } else { + MS_LOG(INFO) << "BuildDatasetGraph end."; + } + return ret; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 350b8650e9f2a4cb32849cdb304f2d177a5bfc38 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:28:11 +0800 Subject: [PATCH 098/160] ADD file via upload --- .../ccsrc/transform/graph_ir/graph_runner.cc | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/graph_runner.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/graph_runner.cc b/mindspore2022/ccsrc/transform/graph_ir/graph_runner.cc new file mode 100644 index 00000000000..a6e6011c013 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/graph_runner.cc @@ -0,0 +1,283 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + */ + +#include "include/transform/graph_ir/graph_runner.h" +#include +#include +#include + +#ifndef ENABLE_LITE_ACL +#include "pybind11/pybind11.h" +#endif +#include "utils/log_adapter.h" +#include "include/common/utils/config_manager.h" +#include "sys/time.h" +#include "include/common/utils/utils.h" +#include "include/common/utils/callbacks.h" +#ifdef ENABLE_D +#include "include/common/utils/callbacks_ge.h" +#endif +#include "utils/ms_context.h" + +#ifndef ENABLE_LITE_ACL +namespace py = pybind11; +#endif +namespace mindspore { +namespace transform { +//在GraphRunner区域定义名为NewSession的函数，返回值类型为std::shared_ptr +/* +首先判断ENABLE_D是否被定义，若是，则执行条件命令： + 定义变量ret和ms_context并令后者储存MsContext::GetInstance()数据 + 判断ms_context是否为空 + 判断ms_context的backend_policy()获取的属性是否为"ge"，若是，则对sess_options进行make_shared并储存到ret中 + 在此基础上在判断ret是否为空，若是则输出EXCEPTION日志"Create GE session failed!" + 输出消息日志"Create new GE session success!"并返回ret，结束条件命令 +若ENABLE_D没有被定义，则输出警告日志"no GE client, return nullptr!"并返回nullptr +*/ +std::shared_ptr GraphRunner::NewSession(const SessionOptions &sess_options) { +#ifdef ENABLE_D + std::shared_ptr ret; + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + ret = std::make_shared(sess_options); + if (ret == nullptr) { + MS_LOG(EXCEPTION) << "Create GE session failed!"; + } + MS_LOG(INFO) << "Create new GE session success!"; + return ret; + } +#endif + + MS_LOG(WARNING) << "no GE client, return nullptr!"; + return nullptr; +} + +//GraphRunner区域的GraphRunner函数 +/* +获取实例的并行策略是否为ONE_DEVICE，若是，则输出消息日志"ME run in ONE_DEVICE strategy mode" +判断option的sess_ptr属性是否为空，若不为空则令sess_ = options.sess_ptr +若为空则用变量属性options.options创建新会话并储存在sess_中 + 在此基础上再判断sess_是否为空，若是则输出警告日志"graph runner sess_ is nullptr!" +*/ +GraphRunner::GraphRunner(const GraphRunnerOptions &options) + : options_(options), graph_manager_(DfGraphManager::GetInstance()) { + if (ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::ONE_DEVICE) { + MS_LOG(INFO) << "ME run in ONE_DEVICE strategy mode"; + } + + if (options.sess_ptr != nullptr) { + sess_ = options.sess_ptr; + } else { + sess_ = NewSession(options.options); + if (sess_ == nullptr) { + MS_LOG(WARNING) << "graph runner sess_ is nullptr!"; + } + } +/* +判断ENABLE_D是否被定义，若是则执行条件命令： + 定义变量ms_context并将获取的实例储存在其中 + 判断ms_context的backend_policy()属性是否为ge，若是则： + 判断会话对于kCheckPoint和CheckpointSaveCallback回叫功能是否实现，若没有实现则输出警告日志"register callback failed!" + 判断会话对于kSummary和SummarySaveCallback回叫功能是否实现，若没有实现则输出警告日志"register summary callback failed!" +结束条件命令 +*/ +#ifdef ENABLE_D + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + // register the callback function注册表的回叫信号功能 + if (sess_->RegisterCallBackFunc(callbacks::kCheckPoint, callbacks::CheckpointSaveCallback) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "register callback failed!"; + } + + if (sess_->RegisterCallBackFunc(callbacks::kSummary, callbacks::SummarySaveCallback) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "register summary callback failed!"; + } + } +#endif +//定义变量wrappers储存graph_manager_.GetAllGraphs()并判断其是否为空，若是，则输出消息日志"The GraphManager is empty!!"并返回 + std::vector wrappers = graph_manager_.GetAllGraphs(); + if (wrappers.empty()) { + MS_LOG(INFO) << "The GraphManager is empty!!"; + return; + } + /* +判断ENABLE_D是否被定义，若是则执行条件命令： + 判断ms_context的backend_policy()属性是否为ge，若不是，则返回 + 利用for循环获取并历遍已储存的图表 + 输出消息日志"Add the graph[*此处为name属性]to GE, it's id is:[*此处为id属性]" + 将id对应的it与graph_manager_进行AddSavedGraphs操作 + 将static_cast(it->id_), *(it->graph_ptr_), it->options_执行AddGraph操作储存在sess_指针中 +结束条件命令 +*/ +#ifdef ENABLE_D + if (ms_context->backend_policy() != "ge") { + return; + } + + for (auto &it : wrappers) { + std::set saved_graph = graph_manager_.GetSavedGraphs(); + auto iter_find = saved_graph.find(std::to_string(it->id_)); + if (iter_find != saved_graph.end()) { + continue; + } + MS_LOG(INFO) << "Add the graph " << (*it).name_ << " to GE, it's id is: " << (*it).id_; + graph_manager_.AddSavedGraphs(std::to_string(it->id_)); + (void)sess_->AddGraph(static_cast(it->id_), *(it->graph_ptr_), it->options_); + } +#endif +} + +//在GraphRunner区域定义名为RunGraph的函数，返回值为自定义类型Status +/* +获取options的name属性储存在name变量中并判断其是否为空，若是则输出错误日志"The graph name is null"并返回return Status::INVALID_ARGUMENT +定义wrap_ptr获取graph的name属性并检验其是否为空，若为空则输出错误日志"Get graph form DfGraphManager failed!"并返回Status::NOT_FOUND +检测wrap_ptr->graph_ptr_是否为空，若为空则输出警告日志"The graph is null"并返回Status::NOT_FOUND +*/ +Status GraphRunner::RunGraph(const RunOptions &options, const std::vector &inputs, + std::vector *outputs) { + std::string name = options.name; + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null"; + return Status::INVALID_ARGUMENT; + } + + DfGraphWrapperPtr wrap_ptr = graph_manager_.GetGraphByName(name); + if (wrap_ptr == nullptr) { + MS_LOG(ERROR) << "Get graph form DfGraphManager failed!"; + return Status::NOT_FOUND; + } + + if (wrap_ptr->graph_ptr_ == nullptr) { + MS_LOG(WARNING) << "The graph is null"; + return Status::NOT_FOUND; + } + + // call ge::RunGraph() to exec a graph;//调试ge::RunGraph()执行图表 + //获取输出的size属性并输出相应消息日志 + //获取开始时间 + std::vector ge_inputs; + std::vector ge_outputs; + + (void)std::transform(inputs.begin(), inputs.end(), std::back_inserter(ge_inputs), + [](const GeTensorPtr &i) { return *i; }); + + MS_LOG(INFO) << "Run the graph in GE with " << ge_inputs.size() << " inputs"; + + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); +/* +检测ENABLE_D是否被定义，若是，则执行条件命令： + 获取实例并储存在ms_context中并检测其是否为空 + 检测ms_context的backend_policy是否为ge，若是则 + 检测sess_是否为空，若是则输出错误日志"The GE session is null, can't run the graph!"并返回Status::FAILED + 执行RunGraph功能并将结果数据储存在ret中 + 检测上一步是否成功，若不成功则输出错误日志"Call GE RunGraph Failed, ret is:[*此处为ret]"并返回Status::FAILED +若ENABLE_D没有被定义，则执行 ge_outputs.swap(ge_inputs) +结束条件命令 +*/ +#ifdef ENABLE_D + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + if (sess_ == nullptr) { + MS_LOG(ERROR) << "The GE session is null, can't run the graph!"; + return Status::FAILED; + } + ge::Status ret = sess_->RunGraph(static_cast(wrap_ptr->id_), ge_inputs, ge_outputs); + if (ret != ge::GRAPH_SUCCESS) { + MS_LOG(ERROR) << "Call GE RunGraph Failed, ret is: " << ret; + return Status::FAILED; + } + } +#else + ge_outputs.swap(ge_inputs); +#endif +//获取结束时间并计算出调试所用时长 +//输出消息日志"Call GE RunGraph Success in "[*此处为消耗时长]" us, the GE outputs num is:[*此处为e_outputs.size]" + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Call GE RunGraph Success in " << cost << " us, the GE outputs num is: " << ge_outputs.size(); + + (void)std::transform(ge_outputs.begin(), ge_outputs.end(), std::back_inserter(*outputs), + [](const GeTensor &ge_tensor) { return std::make_shared(ge_tensor); }); + + return Status::SUCCESS; +} +//在GraphRunner区域定义名为RunGraph的函数，返回值为自定义类型Status +/* +利用for循环分别对inputs的size属性和shape属性进行转存，并分别输出消息日志："inputs tensor's data size is:[*此处为size]" + "inputs tensor's shape is: {[*此处为shape]}" +利用ge_tensor_ptr转存对以it和kOpFormat_NCHW为变量进行ConvertTensor操作之后的结果并检验其是否为空 +若不为空则进行ge_inputs.emplace_back(ge_tensor_ptr)操作 +若为空则输出消息日志"Convert input Me tensor to Ge tensor failed. Abort this graph"并返回Status::FAILED +*/ +Status GraphRunner::RunGraph(const RunOptions &options, const std::vector &inputs, + std::vector *const outputs) { + std::vector ge_inputs; + for (auto it : inputs) { + MS_EXCEPTION_IF_NULL(it); + MS_LOG(INFO) << "inputs tensor's data size is: " << (*it).DataSize(); + auto shape = (*it).shape(); + std::string shape_str; + for (const auto &elem : shape) { + shape_str += std::to_string(elem); + shape_str += " "; + } + MS_LOG(INFO) << "inputs tensor's shape is: { " << shape_str << "}"; + + auto ge_tensor_ptr = TransformUtil::ConvertTensor(it, kOpFormat_NCHW); + if (ge_tensor_ptr != nullptr) { + ge_inputs.emplace_back(ge_tensor_ptr); + } else { + MS_LOG(INFO) << "Convert input Me tensor to Ge tensor failed. Abort this graph"; + return Status::FAILED; + } + } + + std::vector ge_outputs; + Status ret; + { + // Release GIL before calling into (potentially long-running) C++ code//在调试为c++代码前释放GIL +#ifndef ENABLE_LITE_ACL + py::gil_scoped_release release; +#endif + /*使用ret转存对变量options, ge_inputs, &ge_outputs进行RunGraph之后的数据结果并检测其是否成功 + 若不成功则返回ret + 若成功则将it转化为GeTensors并存入tensor并检测tensor是否为空，若不为空则执行(void)outputs->emplace_back(tensor) + 输出消息日志"Return Me tensor outputs num is:[*此处为outputs->size]"并返回Status::SUCCESS + */ + ret = RunGraph(options, ge_inputs, &ge_outputs); + } + if (ret != Status::SUCCESS) { + return ret; + } else { + // convert GeTensor to MeTensor//将GeTensor转化为MeTensor + for (auto &it : ge_outputs) { + auto tensor = TransformUtil::ConvertGeTensor(it); + if (tensor != nullptr) { + (void)outputs->emplace_back(tensor); + } + } + MS_LOG(INFO) << "Return Me tensor outputs num is: " << outputs->size(); + return Status::SUCCESS; + } +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 3b388f572fdd61288e5f6d6c69b65f81b87f01ae Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:28:27 +0800 Subject: [PATCH 099/160] ADD file via upload --- .../ccsrc/transform/graph_ir/io_format_map.cc | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/io_format_map.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/io_format_map.cc b/mindspore2022/ccsrc/transform/graph_ir/io_format_map.cc new file mode 100644 index 00000000000..4fc75537248 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/io_format_map.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/io_format_map.h" + +namespace mindspore { +namespace transform { + //用io_format_map_指针储存数据 +mindspore::HashMap IOFormatMap::io_format_map_ = {{"BasicLSTMCell", "ND"}, + {"BasicLSTMCellInputGrad", "ND"}, + {"BasicLSTMCellCStateGrad", "ND"}, + {"Dequant", "ND"}, + {"DynamicGRUV2", "ND"}, + {"DynamicGRUV2Grad", "ND"}, + {"DynamicRNN", "ND"}, + {"DynamicRNNGrad", "ND"}, + {"MatMul", "ND"}, + {"BatchMatMul", "ND"}, + {"BatchMatMulV2", "ND"}, + {"Quant", "ND"}, + {"BasicLSTMCellWeightGrad", "HWCN"}, + {"ExtractImagePatches", "NCHW"}, + {"Conv3D", "format"}, + {"MaxPool3D", "NCDHW"}, + {"Conv3DBackpropFilter", "format"}, + {"Conv3DBackpropInput", "format"}, + {"Conv3DTranspose", "format"}}; +mindspore::HashMap &IOFormatMap::get() { return io_format_map_; } +//采用返回io_format_map_指针的引用的方式，防止因连续赋值导致不必要的计算开销 +} // namespace transform +} // namespace mindspore -- 2.34.1 From 9776e3ff99b31a6c8c6aafc33a4e0fc226b1945d Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:32:00 +0800 Subject: [PATCH 100/160] ADD file via upload --- .../ccsrc/transform/graph_ir/io_format_map.h | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/io_format_map.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/io_format_map.h b/mindspore2022/ccsrc/transform/graph_ir/io_format_map.h new file mode 100644 index 00000000000..8a3ee4f4980 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/io_format_map.h @@ -0,0 +1,34 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_IO_FORMAT_MAP_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_IO_FORMAT_MAP_H_ + +#include +#include "utils/hash_map.h" + +namespace mindspore { +namespace transform { + //定义IOFormatMap类 +class IOFormatMap { + public: + static mindspore::HashMap &get();//定义方法get()，返回值为对象的引用 + + private: + static mindspore::HashMap io_format_map_;//定义私有指针io_format_map_ +}; +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_IO_FORMAT_MAP_H_ -- 2.34.1 From 21e76296881bf04b9e64cb6e9b9c7e8c041cb401 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:32:45 +0800 Subject: [PATCH 101/160] ADD file via upload --- .../transform/graph_ir/op_adapter_base.h | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_adapter_base.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_adapter_base.h b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_base.h new file mode 100644 index 00000000000..abc50c2b8e5 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_base.h @@ -0,0 +1,205 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_ + +#include +#include +#include +#include +#include + +#include "utils/hash_map.h" +#include "include/transform/graph_ir/util.h" +#include "ir/anf.h" +#include "ir/primitive.h" +#include "ir/value.h" +#include "include/transform/graph_ir/types.h" +#include "graph/operator_reg.h" +#include "external/ge/ge_api.h" +#include "graph/tensor.h" + +namespace ge { +class CustomOperator : public Operator { + public: + CustomOperator(const string &name, const string &type) : Operator(name, type) {} + + ~CustomOperator() override{}; + + void CustomInputRegister(const string &name) { Operator::InputRegister(name); } + + void CustomOutputRegister(const string &name) { Operator::OutputRegister(name); } + + void CustomInferFuncRegister(const std::function &func) { + Operator::InferFuncRegister(func); + } +}; +} // namespace ge + +namespace mindspore { +namespace transform { +using CusOperatorPtr = std::shared_ptr; +using CustomOperator = ge::CustomOperator; +using AttrFunc = std::function; +using OutputFunc = std::function; +using InputOpFunc = std::function; +using InputHandleFunc = std::function; +using CreateDynInputOpFunc = std::function; +using DynInputOpFunc = std::function; +using DynInputHandleFunc = std::function; +using UpdateOutputDescFunc = std::function; +using CreateDynOutputOpFunc = std::function; +using CreateDynSubGraphFunc = std::function; +using DynSubGraphFunc = std::function; + +//定义结构体AttrDesc，包含string型变量name和AttrFunc型变量set_attr +struct AttrDesc { + std::string name; + AttrFunc set_attr; +}; + +/*定义结构体InputDesc，包含string型变量name; + InputOpFunc型变量set_op; + InputHandleFunc型变量set_handle; + UpdateOutputDescFunc型变量update_input_desc; +*/ +struct InputDesc { + std::string name; + InputOpFunc set_op; + InputHandleFunc set_handle; + UpdateOutputDescFunc update_input_desc; +}; +/*定义结构体DynInputDesc，包含string型变量name; + CreateDynInputOpFunc型变量create_dyn_input; + DynInputOpFunc型变量set_op; + DynInputHandleFunc型变量set_handle; +*/ +struct DynInputDesc { + std::string name; + CreateDynInputOpFunc create_dyn_input; + DynInputOpFunc set_op; + DynInputHandleFunc set_handle; +}; +/*定义结构体DynSubGraphDesc，包含string型变量name; + CreateDynSubGraphFunc型变量create_dyn_subgraph; + DynSubGraphFunc型变量set_subgraph; +*/ +struct DynSubGraphDesc { + std::string name; + CreateDynSubGraphFunc create_dyn_subgraph; + DynSubGraphFunc set_subgraph; +}; +/*定义结构体OutputDesc，包含string型变量name; + UpdateOutputDescFunc型变量update_out_desc; +*/ +struct OutputDesc { + std::string name; + UpdateOutputDescFunc update_out_desc; +}; +/*定义结构体DynOutputDesc，包含string型变量name; + CreateDynOutputOpFunc型变量create_dyn_output; +*/ +struct DynOutputDesc { + std::string name; + CreateDynOutputOpFunc create_dyn_output; +}; +//创建BaseOpAdapter类 +/* +创建并定义属于BaseOpAdapter类中的函数，其功能包括：生成OperatorPtr类型变量 + 设置sub图表 + 设置输入对象 + 设置Attr + 获取输出对象 + 更新输出数据 + 获取输入的map + 获取输入的attrmap + 将attr添加到drawgraph + 从drawgraph获取attr + 清空attr的vect属性 + 等功能，其具体的实例将在其他属于adapter部分的文件中进一步分析 +*/ +class BaseOpAdapter { + public: + virtual ~BaseOpAdapter() {} + virtual OperatorPtr generate(const AnfNodePtr &anf) = 0; + virtual OperatorPtr generate(const std::string &type) { return std::make_shared(type); } + virtual int setSubgraph(const OperatorPtr &op, int index, const std::shared_ptr> &branches) = 0; + virtual int setInput(const OperatorPtr &op, int index, const OperatorPtr &input) = 0; + virtual int setInput(const OperatorPtr &op, int index, const OutHandler &handle) = 0; + virtual int setInput(const OperatorPtr &op, int index, + const std::shared_ptr> &handler_vec) = 0; + virtual int setAttr(const OperatorPtr &op, const std::string &attrKey, const ValuePtr &attrValue) = 0; + virtual int setAttr(const OperatorPtr &op, const PrimitivePtr &prim) = 0; + virtual int setAttr(const OperatorPtr &op, const AnfNodePtr &node) = 0; + virtual mindspore::HashMap GetExtraAttr() = 0; + template ::value>::type> + int setAttr(const OperatorPtr &op, const std::string &attrKey, const std::shared_ptr &attrValue) { + return setAttr(op, attrKey, MakeValue(attrValue)); + } + template ::value>::type> + int setAttr(const OperatorPtr &op, const std::string &attrKey, const T &attrValue) { + return setAttr(op, attrKey, MakeValue(attrValue)); + } + virtual OutHandler getOutput(const OperatorPtr &op, int index) = 0; + virtual void updateOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const AnfNodePtr &node) = 0; + virtual const mindspore::HashMap &getInputMap() = 0; + virtual const mindspore::HashMap &getInputAttrMap() = 0; + virtual const mindspore::HashMap &getDynInputMap() = 0; + virtual const mindspore::HashMap &getOutputMap() = 0; + virtual const mindspore::HashMap &getDynSubgraphMap() = 0; + void AddAttrToDrawGraph(const std::string &attr_str) { attrs_vec_.push_back(attr_str); } + const std::vector &GetAttrsFromDrawGraph() const { return attrs_vec_; } + void clearAttrVect() { attrs_vec_.clear(); } + + private: + std::vector attrs_vec_; +}; + +using OpAdapterPtr = std::shared_ptr; + +enum AttrType { + ATTR_INT = 0, + ATTR_FLOAT, + ATTR_DOUBLE, + ATTR_STRING, + ATTR_TENSOR, + ATTR_BOOL, + ATTR_LIST_INT, + ATTR_LIST_ANY_INT, + ATTR_ENUM +}; + +struct GeEnum {}; +struct TFType {}; +struct GEType {}; + +// declare Any type +template +struct AnyTraits { + using type = T; +}; + +template <> +struct AnyTraits { + using type = int64_t; +}; + +using ExtraAttr = mindspore::HashMap; +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_ -- 2.34.1 From 68149376fbd32eb86ec6cf46459389834222e2aa Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:33:01 +0800 Subject: [PATCH 102/160] ADD file via upload --- .../transform/graph_ir/op_adapter_desc.h | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_adapter_desc.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_adapter_desc.h b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_desc.h new file mode 100644 index 00000000000..a9bf9a585ec --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_desc.h @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_ + +#include +#include "transform/graph_ir/op_adapter.h" + +namespace mindspore { +namespace transform { + //创建OpAdapterDesc类 + /* + 定义OpAdapterDesc，根据参数不同对应不同的方法，包括根据输入改对象的train_和infer_属性或令其为空 + */ +class OpAdapterDesc { + public: + OpAdapterDesc() : train_(nullptr), infer_(nullptr) {} + + OpAdapterDesc(const OpAdapterPtr &train, const OpAdapterPtr &infer) : train_(train), infer_(infer) {} + + explicit OpAdapterDesc(const OpAdapterPtr &common) : train_(common), infer_(common) {} + + OpAdapterDesc(const OpAdapterDesc &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + } + + OpAdapterDesc(OpAdapterDesc &&desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + desc.train_ = nullptr; + desc.infer_ = nullptr; + } + + ~OpAdapterDesc() = default; + + OpAdapterPtr Get(bool train) const { return train ? train_ : infer_; } +//定义名为operator的函数返回值类型为OpAdapterDesc，主要执行train_和infer_变量的修改 + OpAdapterDesc &operator=(const OpAdapterDesc &desc) { + if (this != &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + } + return *this; + } +//定义名为operator的函数返回值类型为OpAdapterDesc，主要执行train_和infer_变量的修改以及属于desc的train_和infer_变量的清空 + OpAdapterDesc &operator=(OpAdapterDesc &&desc) { + if (this != &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + desc.train_ = nullptr; + desc.infer_ = nullptr; + } + return *this; + } +//定义私有化OpAdapterPtr类型变量train_和infer_ + private: + OpAdapterPtr train_; + OpAdapterPtr infer_; +}; + +using OpAdapterDescPtr = std::shared_ptr; +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_ -- 2.34.1 From d13c7a7625e4080948a0da478a7bc2ae2af73b15 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:33:20 +0800 Subject: [PATCH 103/160] ADD file via upload --- .../transform/graph_ir/op_adapter_map.cc | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_adapter_map.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_adapter_map.cc b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_map.cc new file mode 100644 index 00000000000..707fff196e0 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_map.cc @@ -0,0 +1,36 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "include/transform/graph_ir/op_adapter_map.h" +#include +#include "graph/operator.h" +#include "transform/graph_ir/op_adapter_desc.h" + +namespace mindspore { +namespace transform { +//定义adpt_map_存储的数据 +namespace { +mindspore::HashMap adpt_map_ = { + {kNameCustomOp, std::make_shared(std::make_shared>())}}; +} // namespace +//分别定义指针cus_input_map_和cus_output_map_ +template <> +mindspore::HashMap> OpAdapter::cus_input_map_{}; +template <> +mindspore::HashMap> OpAdapter::cus_output_map_{}; +//定义get的方法体为返回adpt_map_ +mindspore::HashMap &OpAdapterMap::get() { return adpt_map_; } +} // namespace transform +} // namespace mindspore -- 2.34.1 From 7f8537ca4c9f164ed14785507f432773cb23e7a6 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:34:16 +0800 Subject: [PATCH 104/160] ADD file via upload --- .../transform/graph_ir/op_adapter_util.cc | 359 ++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.cc diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.cc b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.cc new file mode 100644 index 00000000000..9088081b052 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.cc @@ -0,0 +1,359 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_adapter_util.h" + +#include +#include +#include + +#include "include/common/utils/utils.h" +#include "utils/check_convert_utils.h" +#include "transform/graph_ir/op_adapter_base.h" +#include "transform/graph_ir/io_format_map.h" + +namespace mindspore { +namespace transform { +//定义名为ConvertAnyUtil的函数，返回值类型为GeTensor +/*将me_tensor转化为ge_tensor +检验其是否为空并返回ge_tensor*/ +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits &) { + // To-DO the format may read from ME tensor//执行计划可能会从Me tensor中读取 + MS_EXCEPTION_IF_NULL(value); + auto me_tensor = value->cast(); + auto ge_tensor = TransformUtil::ConvertTensor(me_tensor, kOpFormat_ND); + return ge_tensor == nullptr ? GeTensor() : *ge_tensor; +} +//定义名为ConvertAnyUtil的函数，返回值类型为std::vector +/* +检验value是否为空，引入list +检验name是否为pad，若是，则 + 判断value与isa()是否相匹配，若不相配则输出EXCEPTION日志"Value should be ValueTuple, but got[*此处为value->type_name()]" + 定义vec执行value->cast() + 重设list的size并对其开始的两位赋值 + */ +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &name, + const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); + std::vector list; + if (name == "pad") { + if (!value->isa()) { + MS_LOG(EXCEPTION) << "Value should be ValueTuple, but got" << value->type_name(); + } + auto vec = value->cast(); + list.resize(vec->value().size() + 2); + list[0] = 1; + list[1] = 1; + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin() + 2, + [](const ValuePtr &val) { return static_cast(GetValue(val)); }); + } + /*若name不为pad，则定义data获取value，设定size的值并将int转化为list*/ + else { + int64_t data = GetValue(value); + int size = 2; // 2 int in list//list中有两个int型 + list = TransformUtil::ConvertIntToList(data, size); + } + //返回list + return list; +} +//定义名为ConvertAnyUtil的函数，返回值类型为std::string +/* +判断value是否为空 +定义vec储存value->cast()所得数据并判断vec是否为空，若是，则输出EXCEPTION日志"not ValueTuplePtr" +利用buffer储存value并用","隔开 +返回buffer.str() +*/ +std::string ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); + if (vec == nullptr) { + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::ostringstream buffer; + int i = 0; + for (auto &it : vec->value()) { + if (i != 0) { + buffer << ","; + } + buffer << GetValue(it); + i++; + } + return buffer.str(); +} +//定义名为ConvertAnyUtil的函数，返回值类型为std::vector +/* +判断value是否为空 +定义vec储存value->cast()所得数据并判断vec是否为空，若是，则输出EXCEPTION日志"not ValueTuplePtr" +定义list，并重设其size属性 +返回list +*/ +std::vector ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); + if (vec == nullptr) { + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::vector list; + list.resize(vec->value().size()); + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin(), + [](const ValuePtr &val) { return static_cast(GetValue(val)); }); + return list; +} +//定义名为ConvertAnyUtil的函数，返回值类型为std::vector +/* +判断value是否为空 +定义vec储存value->cast()所得数据并判断vec是否为空，若是，则输出EXCEPTION日志"not ValueTuplePtr" +定义list，并重设其size属性 +判断format是否为kOpFormat_NHWC，若是则进一步判断list的size属性是否小于四，若是，则输出对应EXCEPTION日志 +若不是，则给list中的每个位置赋值 +返回list +*/ +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &format, + const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); + if (vec == nullptr) { + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::vector list; + list.resize(vec->value().size()); + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin(), + [](const ValuePtr &val) { return static_cast(GetValue(val)); }); + if (format == kOpFormat_NHWC) { + if (list.size() < 4) { + MS_LOG(EXCEPTION) << "The size of list is less than 4"; + } else { + int64_t temp = list[1]; + list[1] = list[2]; + list[2] = list[3]; + list[3] = temp; + } + } + return list; +} + +GeDataType ConvertAnyUtil(const ValuePtr &value, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + if (!value->isa()) { + MS_LOG(EXCEPTION) << "error convert Value to TypePtr for value: " << value->ToString() + << ", type: " << value->type_name() << ", value should be a Typeptr"; + } + auto type = value->cast(); + MS_EXCEPTION_IF_NULL(type); + TypeId me_type = type->type_id(); + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(type)->element()->type_id(); + } + return TransformUtil::ConvertDataType(me_type); +} + +GeTensor VectorToTensorUtil(const ValuePtr &value) { + // convert tuple or list to ge tensor, only supported one dim for now + //将tuple或者list转化为ge_tensor，当前只支持一个dim + MS_EXCEPTION_IF_NULL(value); + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + if (vec.empty()) { + MS_LOG(WARNING) << "Convert a none tuple to an empty ge tensor"; + return GeTensor(GeTensorDesc(ge::Shape({0}))); + } + MS_EXCEPTION_IF_NULL(vec[0]); + if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Int32"; + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeInt32, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(int32_t)); + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Int64"; + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeInt64, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(int64_t)); + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Float32"; + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeFloat32, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(float)); + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Bool"; + // We use uint8_t to save bool type data + //我们用uin8_t来保存bool类型的数据 + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeBool, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, static_cast(data.data()), data.size() * sizeof(uint8_t)); + } else { + MS_LOG(EXCEPTION) << "Unsupported data type of tuple or list elements: " << vec[0]->type_name(); + } +} + +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + if (value->isa()) { + // convert me tensor to ge tensor + //将me_tensor转化为ge_tensor + return ConvertAnyUtil(value, AnyTraits()); + } else if (value->isa() || value->isa()) { + return VectorToTensorUtil(value); + } else if (value->isa()) { + // convert scalar Int to GeTensor + //将scalar Int转化为GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Int32"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT32); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(int32_t)); + } else if (value->isa()) { + // convert scalar Int64 to GeTensor + //将scalar转化为GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Int64"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(int64_t)); + } else if (value->isa()) { + // convert scalar FP32 to GeTensor + //将scalar FP32转化为GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = FP32"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_FLOAT); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(float)); + } else if (value->isa()) { + // convert scalar FP32 to GeTensor + //将scalar FP32转化为GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Bool"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_BOOL); + auto v = GetValue(value); + desc.SetRealDimCnt(0); + return GeTensor(desc, reinterpret_cast(&v), sizeof(bool)); + } else if (value->isa()) { + // convert String to GeTensor + //将String转化为GeTensor + MS_LOG(INFO) << "convert string to tensor with data type = String"; + std::string v = GetValue(value); + std::vector ge_shape; + GeShape shape(ge_shape); + GeTensorDesc desc(shape, ge::FORMAT_NCHW, ge::DT_STRING); + GeTensor str_tensor(desc); + (void)str_tensor.SetData(v); + return str_tensor; + } else { + MS_LOG(WARNING) << "Unsupported value type: " << value->type_name() + << " to convert to tensor. Value: " << value->ToString(); + } + return GeTensor(); +} + +bool IsCustomPrim(const PrimitivePtr &prim) { + if (prim == nullptr) { + return false; + } + + ValuePtr flag = prim->GetAttr("_custom_op_flag"); + if (flag == nullptr) { + return false; + } + + bool is_custom_op = GetValue(flag); + if (!is_custom_op && prim->GetAttr("_custom_op_impl_config_path") != nullptr) { + MS_LOG(EXCEPTION) << "The custom op flag is false, but the op information config path is not null, non-custom op " + "can not assign the op information config path."; + } + + return is_custom_op; +} + +bool IsCustomCNode(const AnfNodePtr &anf) { + if (anf == nullptr) { + return false; + } + auto node = anf->cast(); + if (node == nullptr) { + return false; + } + if (node->inputs().empty()) { + MS_LOG(EXCEPTION) << "Length of node inputs is empty"; + } + MS_EXCEPTION_IF_NULL(node->inputs()[0]); + if (!node->inputs()[0]->isa()) { + return false; + } + auto cus_prim = GetValueNode(node->inputs()[0]); + if (cus_prim == nullptr) { + return false; + } + + return IsCustomPrim(cus_prim); +} + +std::string GetOpIOFormat(const AnfNodePtr &anf) { + std::string ret; + if (anf == nullptr) { + MS_LOG(ERROR) << "The anf is nullptr"; + return ret; + } + auto node = anf->cast(); + if (node == nullptr) { + MS_LOG(ERROR) << "The anf is not a cnode."; + return ret; + } + if (node->inputs().empty()) { + MS_LOG(EXCEPTION) << "Length of node inputs is empty."; + } + MS_EXCEPTION_IF_NULL(node->inputs()[0]); + if (!node->inputs()[0]->isa()) { + MS_LOG(ERROR) << "The anf is not a value node."; + return ret; + } + auto prim = GetValueNode(node->inputs()[0]); + if (prim == nullptr) { + MS_LOG(ERROR) << "The anf is not a Primitive."; + return ret; + } + if (prim->HasAttr("io_format")) { + return GetValue(prim->GetAttr("io_format")); + } + auto io_format_map = IOFormatMap::get(); + auto iter = io_format_map.find(prim->name()); + if (iter == io_format_map.end()) { + return "NCHW"; + } + if (iter->second == "format") { + ValuePtr format = prim->GetAttr("format"); + MS_EXCEPTION_IF_NULL(format); + if (format->isa()) { + bool converted = CheckAndConvertUtils::ConvertAttrValueToString(prim->name(), "format", &format); + if (converted) { + return GetValue(format); + } + } else { + return GetValue(format); + } + } + return iter->second; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 48b7d585734a70039c8eee68e2d993b7d7194ca4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 18:36:29 +0800 Subject: [PATCH 105/160] ADD file via upload --- .../transform/graph_ir/op_adapter_util.h | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.h diff --git a/mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.h b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.h new file mode 100644 index 00000000000..b4ef182c5c1 --- /dev/null +++ b/mindspore2022/ccsrc/transform/graph_ir/op_adapter_util.h @@ -0,0 +1,77 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_ + +#include +#include + +#include "transform/graph_ir/op_adapter_base.h" + +namespace mindspore { +namespace transform { +template +static Q ConvertAnyUtil(const ValuePtr &value, const AnyTraits

&, const AnyTraits &) { + return static_cast(GetValue

(value)); +} + +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits &traits); + +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &name, + const AnyTraits>); + +std::string ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits); + +std::vector ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits); + +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &format, + const AnyTraits>, const AnyTraits); + +GeDataType ConvertAnyUtil(const ValuePtr &value, const AnyTraits); + +template +//定义名为ConvertAnyUtil的函数，返回值类型为std::vector +/* +判断value是否为空 +若value既非isa()也非isa()则输出EXCEPTION日志 +判断value是否为isa()，若是则执行value->cast()->value()并将结果储存在vec中 +定义变量data并将对it执类型转换后添加到data的尾部 +返回data +*/ +std::vector ConvertAnyUtil(const ValuePtr &value, AnyTraits

, const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); + if (!value->isa() && !value->isa()) { + MS_LOG(EXCEPTION) << "error convert Value to vector for value: " << value->ToString() + << ", type: " << value->type_name() << ", value should be a tuple or list"; + } + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + std::vector data; + for (auto &it : vec) { + data.push_back(ConvertAnyUtil(it, AnyTraits

(), AnyTraits())); + } + return data; +} +//转换value的数据类型 +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits); +//判断prim是否为CustomPrim +//判断node是否为CustomCNode +bool IsCustomPrim(const PrimitivePtr &prim); +bool IsCustomCNode(const AnfNodePtr &node); +std::string GetOpIOFormat(const AnfNodePtr &node); +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_ -- 2.34.1 From 772db716b7a6f0d72832b56dbfda96f6368dfca0 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:53:05 +0800 Subject: [PATCH 106/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc' --- .../mindrecord/include/meta/shard_category.cc | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc deleted file mode 100644 index 462023d09b6..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_category.h" - -namespace mindspore { -namespace mindrecord { -ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, - bool replacement) - : categories_(categories), - category_field_(""), - num_elements_(num_elements), - num_categories_(0), - replacement_(replacement) {} - -ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, - bool replacement) - : categories_({}), - category_field_(category_field), - num_elements_(num_elements), - num_categories_(num_categories), - replacement_(replacement) {} - -Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); } - -int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (dataset_size == 0) return dataset_size; - if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { - num_classes = std::min(num_categories_, num_classes); - if (num_classes == 0) { - return 0; - } - if (num_elements_ > std::numeric_limits::max() / num_classes) { - return -1; - } - return num_classes * num_elements_; - } - return 0; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From e4650acc665b0a931e1adee2f6036206d902cdfc Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:53:32 +0800 Subject: [PATCH 107/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc' --- .../mindrecord/include/meta/shard_column.cc | 500 ------------------ 1 file changed, 500 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc deleted file mode 100644 index 9626acd9404..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_column.cc +++ /dev/null @@ -1,500 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_column.h" - -#include "utils/ms_utils.h" -#include "minddata/mindrecord/include/common/shard_utils.h" -#include "minddata/mindrecord/include/shard_error.h" - -namespace mindspore { -namespace mindrecord { -ShardColumn::ShardColumn(const std::shared_ptr &shard_header, bool compress_integer) { - auto first_schema = shard_header->GetSchemas()[0]; - json schema_json = first_schema->GetSchema(); - Init(schema_json, compress_integer); -} - -ShardColumn::ShardColumn(const json &schema_json, bool compress_integer) { Init(schema_json, compress_integer); } - -void ShardColumn::Init(const json &schema_json, bool compress_integer) { - auto schema = schema_json["schema"]; - auto blob_fields = schema_json["blob_fields"]; - - bool has_integer_array = false; - for (json::iterator it = schema.begin(); it != schema.end(); ++it) { - const std::string &column_name = it.key(); - column_name_.push_back(column_name); - - json it_value = it.value(); - - std::string str_type = it_value["type"]; - column_data_type_.push_back(ColumnDataTypeMap.at(str_type)); - if (it_value.find("shape") != it_value.end()) { - std::vector vec(it_value["shape"].size()); - std::copy(it_value["shape"].begin(), it_value["shape"].end(), vec.begin()); - column_shape_.push_back(vec); - if (str_type == "int32" || str_type == "int64") { - has_integer_array = true; - } - } else { - std::vector vec = {}; - column_shape_.push_back(vec); - } - } - - for (uint64_t i = 0; i < column_name_.size(); i++) { - column_name_id_[column_name_[i]] = i; - } - - for (const auto &field : blob_fields) { - blob_column_.push_back(field); - } - - for (uint64_t i = 0; i < blob_column_.size(); i++) { - blob_column_id_[blob_column_[i]] = i; - } - - has_compress_blob_ = (compress_integer && has_integer_array); - num_blob_column_ = blob_column_.size(); -} - -Status ShardColumn::GetColumnTypeByName(const std::string &column_name, ColumnDataType *column_data_type, - uint64_t *column_data_type_size, std::vector *column_shape, - ColumnCategory *column_category) { - RETURN_UNEXPECTED_IF_NULL(column_data_type); - RETURN_UNEXPECTED_IF_NULL(column_data_type_size); - RETURN_UNEXPECTED_IF_NULL(column_shape); - RETURN_UNEXPECTED_IF_NULL(column_category); - // Skip if column not found - *column_category = CheckColumnName(column_name); - CHECK_FAIL_RETURN_UNEXPECTED(*column_category != ColumnNotFound, - "[Internal ERROR] the type of column: " + column_name + " can not found."); - - // Get data type and size - auto column_id = column_name_id_[column_name]; - *column_data_type = column_data_type_[column_id]; - *column_data_type_size = ColumnDataTypeSize[*column_data_type]; - *column_shape = column_shape_[column_id]; - return Status::OK(); -} - -Status ShardColumn::GetColumnValueByName(const std::string &column_name, const std::vector &columns_blob, - const json &columns_json, const unsigned char **data, - std::unique_ptr *data_ptr, uint64_t *const n_bytes, - ColumnDataType *column_data_type, uint64_t *column_data_type_size, - std::vector *column_shape) { - RETURN_UNEXPECTED_IF_NULL(column_data_type); - RETURN_UNEXPECTED_IF_NULL(column_data_type_size); - RETURN_UNEXPECTED_IF_NULL(column_shape); - // Skip if column not found - auto column_category = CheckColumnName(column_name); - CHECK_FAIL_RETURN_UNEXPECTED(column_category != ColumnNotFound, - "[Internal ERROR] the type of column: " + column_name + " can not found."); - // Get data type and size - auto column_id = column_name_id_[column_name]; - *column_data_type = column_data_type_[column_id]; - *column_data_type_size = ColumnDataTypeSize[*column_data_type]; - *column_shape = column_shape_[column_id]; - - // Retrieve value from json - if (column_category == ColumnInRaw) { - RETURN_IF_NOT_OK(GetColumnFromJson(column_name, columns_json, data_ptr, n_bytes)); - *data = reinterpret_cast(data_ptr->get()); - return Status::OK(); - } - - // Retrieve value from blob - RETURN_IF_NOT_OK(GetColumnFromBlob(column_name, columns_blob, data, data_ptr, n_bytes)); - if (*data == nullptr) { - *data = reinterpret_cast(data_ptr->get()); - } - return Status::OK(); -} - -Status ShardColumn::GetColumnFromJson(const std::string &column_name, const json &columns_json, - std::unique_ptr *data_ptr, uint64_t *n_bytes) { - RETURN_UNEXPECTED_IF_NULL(n_bytes); - RETURN_UNEXPECTED_IF_NULL(data_ptr); - auto column_id = column_name_id_[column_name]; - auto column_data_type = column_data_type_[column_id]; - - // Initialize num bytes - *n_bytes = ColumnDataTypeSize[column_data_type]; - auto json_column_value = columns_json[column_name]; - CHECK_FAIL_RETURN_UNEXPECTED(json_column_value.is_string() || json_column_value.is_number(), - "[Internal ERROR] the value of column: " + column_name + - " should be string or number but got: " + json_column_value.dump()); - switch (column_data_type) { - case ColumnFloat32: { - return GetFloat(data_ptr, json_column_value, false); - } - case ColumnFloat64: { - return GetFloat(data_ptr, json_column_value, true); - } - case ColumnInt32: { - return GetInt(data_ptr, json_column_value); - } - case ColumnInt64: { - return GetInt(data_ptr, json_column_value); - } - default: { - // Convert string to c_str - std::string tmp_string; - if (json_column_value.is_string()) { - tmp_string = json_column_value.get(); - } else { - tmp_string = json_column_value.dump(); - } - *n_bytes = tmp_string.size(); - auto data = reinterpret_cast(common::SafeCStr(tmp_string)); - *data_ptr = std::make_unique(*n_bytes); - for (uint32_t i = 0; i < *n_bytes; i++) { - (*data_ptr)[i] = *(data + i); - } - break; - } - } - return Status::OK(); -} - -template -Status ShardColumn::GetFloat(std::unique_ptr *data_ptr, const json &json_column_value, - bool use_double) { - RETURN_UNEXPECTED_IF_NULL(data_ptr); - std::unique_ptr array_data = std::make_unique(1); - if (json_column_value.is_number()) { - array_data[0] = json_column_value; - } else { - // Convert string to float - try { - if (use_double) { - array_data[0] = json_column_value.get(); - } else { - array_data[0] = json_column_value.get(); - } - } catch (json::exception &e) { - RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + json_column_value.dump() + - " to type float, " + std::string(e.what())); - } - } - - auto data = reinterpret_cast(array_data.get()); - *data_ptr = std::make_unique(sizeof(T)); - for (uint32_t i = 0; i < sizeof(T); i++) { - (*data_ptr)[i] = *(data + i); - } - return Status::OK(); -} - -template -Status ShardColumn::GetInt(std::unique_ptr *data_ptr, const json &json_column_value) { - RETURN_UNEXPECTED_IF_NULL(data_ptr); - std::unique_ptr array_data = std::make_unique(1); - int64_t temp_value; - bool less_than_zero = false; - - if (json_column_value.is_number_integer()) { - const json json_zero = 0; - if (json_column_value < json_zero) { - less_than_zero = true; - } - temp_value = json_column_value; - } else if (json_column_value.is_string()) { - std::string string_value = json_column_value; - try { - if (!string_value.empty() && string_value[0] == '-') { - temp_value = std::stoll(string_value); - less_than_zero = true; - } else { - temp_value = static_cast(std::stoull(string_value)); - } - } catch (std::invalid_argument &e) { - RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " + - std::string(e.what())); - } catch (std::out_of_range &e) { - RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " + - std::string(e.what())); - } - } else { - RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value should be type string or number but got: " + - json_column_value.dump()); - } - - if ((less_than_zero && temp_value < static_cast(std::numeric_limits::min())) || - (!less_than_zero && static_cast(temp_value) > static_cast(std::numeric_limits::max()))) { - RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value: " + std::to_string(temp_value) + " is out of range."); - } - array_data[0] = static_cast(temp_value); - - auto data = reinterpret_cast(array_data.get()); - *data_ptr = std::make_unique(sizeof(T)); - for (uint32_t i = 0; i < sizeof(T); i++) { - (*data_ptr)[i] = *(data + i); - } - return Status::OK(); -} - -Status ShardColumn::GetColumnFromBlob(const std::string &column_name, const std::vector &columns_blob, - const unsigned char **data, std::unique_ptr *data_ptr, - uint64_t *const n_bytes) { - RETURN_UNEXPECTED_IF_NULL(data); - uint64_t offset_address = 0; - auto column_id = column_name_id_[column_name]; - RETURN_IF_NOT_OK(GetColumnAddressInBlock(column_id, columns_blob, n_bytes, &offset_address)); - auto column_data_type = column_data_type_[column_id]; - if (has_compress_blob_ && column_data_type == ColumnInt32) { - RETURN_IF_NOT_OK(UncompressInt(column_id, data_ptr, columns_blob, n_bytes, offset_address)); - } else if (has_compress_blob_ && column_data_type == ColumnInt64) { - RETURN_IF_NOT_OK(UncompressInt(column_id, data_ptr, columns_blob, n_bytes, offset_address)); - } else { - *data = reinterpret_cast(&(columns_blob[offset_address])); - } - - return Status::OK(); -} - -ColumnCategory ShardColumn::CheckColumnName(const std::string &column_name) { - auto it_column = column_name_id_.find(column_name); - if (it_column == column_name_id_.end()) { - return ColumnNotFound; - } - auto it_blob = blob_column_id_.find(column_name); - return it_blob == blob_column_id_.end() ? ColumnInRaw : ColumnInBlob; -} - -std::vector ShardColumn::CompressBlob(const std::vector &blob, int64_t *compression_size) { - // Skip if no compress columns - *compression_size = 0; - if (!CheckCompressBlob()) { - return blob; - } - - std::vector dst_blob; - uint64_t i_src = 0; - for (int64_t i = 0; i < num_blob_column_; i++) { - // Get column data type - auto src_data_type = column_data_type_[column_name_id_[blob_column_[i]]]; - auto int_type = src_data_type == ColumnInt32 ? kInt32Type : kInt64Type; - - // Compress and return is blob has 1 column only - if (num_blob_column_ == 1) { - dst_blob = CompressInt(blob, int_type); - *compression_size = static_cast(blob.size()) - static_cast(dst_blob.size()); - return dst_blob; - } - - // Just copy and continue if column dat type is not int32/int64 - uint64_t num_bytes = BytesBigToUInt64(blob, i_src, kInt64Type); - if (src_data_type != ColumnInt32 && src_data_type != ColumnInt64) { - dst_blob.insert(dst_blob.end(), blob.begin() + i_src, blob.begin() + i_src + kInt64Len + num_bytes); - i_src += kInt64Len + num_bytes; - continue; - } - - // Get column slice in source blob - std::vector blob_slice(blob.begin() + i_src + kInt64Len, blob.begin() + i_src + kInt64Len + num_bytes); - // Compress column - auto dst_blob_slice = CompressInt(blob_slice, int_type); - // Get new column size - auto new_blob_size = UIntToBytesBig(dst_blob_slice.size(), kInt64Type); - // Append new column size - dst_blob.insert(dst_blob.end(), new_blob_size.begin(), new_blob_size.end()); - // Append new column data - dst_blob.insert(dst_blob.end(), dst_blob_slice.begin(), dst_blob_slice.end()); - i_src += kInt64Len + num_bytes; - } - MS_LOG(DEBUG) << "Compress blob data from " << blob.size() << " to " << dst_blob.size() << "."; - *compression_size = static_cast(blob.size()) - static_cast(dst_blob.size()); - return dst_blob; -} - -vector ShardColumn::CompressInt(const vector &src_bytes, const IntegerType &int_type) { - uint64_t i_size = kUnsignedOne << static_cast(int_type); - // Get number of elements - uint64_t src_n_int = src_bytes.size() / i_size; - // Calculate bitmap size (bytes) - uint64_t bitmap_size = (src_n_int + kNumDataOfByte - 1) / kNumDataOfByte; - - // Initialize destination blob, more space than needed, will be resized - vector dst_bytes(kBytesOfColumnLen + bitmap_size + src_bytes.size(), 0); - - // Write number of elements to destination blob - vector size_by_bytes = UIntToBytesBig(src_n_int, kInt32Type); - for (uint64_t n = 0; n < kBytesOfColumnLen; n++) { - dst_bytes[n] = size_by_bytes[n]; - } - - // Write compressed int - uint64_t i_dst = kBytesOfColumnLen + bitmap_size; - for (uint64_t i = 0; i < src_n_int; i++) { - // Initialize destination data type - IntegerType dst_int_type = kInt8Type; - // Shift to next int position - uint64_t pos = i * (kUnsignedOne << static_cast(int_type)); - // Narrow down this int - int64_t i_n = BytesLittleToMinIntType(src_bytes, pos, int_type, &dst_int_type); - - // Write this int to destination blob - uint64_t u_n = *reinterpret_cast(&i_n); - auto temp_bytes = UIntToBytesLittle(u_n, dst_int_type); - for (uint64_t j = 0; j < (kUnsignedOne << static_cast(dst_int_type)); j++) { - dst_bytes[i_dst++] = temp_bytes[j]; - } - - // Update date type in bit map - dst_bytes[i / kNumDataOfByte + kBytesOfColumnLen] |= - (static_cast(dst_int_type) << (kDataTypeBits * (kNumDataOfByte - kUnsignedOne - (i % kNumDataOfByte)))); - } - // Resize destination blob - dst_bytes.resize(i_dst); - MS_LOG(DEBUG) << "Compress blob field from " << src_bytes.size() << " to " << dst_bytes.size() << "."; - return dst_bytes; -} - -Status ShardColumn::GetColumnAddressInBlock(const uint64_t &column_id, const std::vector &columns_blob, - uint64_t *num_bytes, uint64_t *shift_idx) { - RETURN_UNEXPECTED_IF_NULL(num_bytes); - RETURN_UNEXPECTED_IF_NULL(shift_idx); - if (num_blob_column_ == 1) { - *num_bytes = columns_blob.size(); - *shift_idx = 0; - return Status::OK(); - } - auto blob_id = blob_column_id_[column_name_[column_id]]; - - for (int32_t i = 0; i < blob_id; i++) { - *shift_idx += kInt64Len + BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type); - } - *num_bytes = BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type); - - (*shift_idx) += kInt64Len; - - return Status::OK(); -} - -template -Status ShardColumn::UncompressInt(const uint64_t &column_id, std::unique_ptr *const data_ptr, - const std::vector &columns_blob, uint64_t *num_bytes, uint64_t shift_idx) { - RETURN_UNEXPECTED_IF_NULL(data_ptr); - RETURN_UNEXPECTED_IF_NULL(num_bytes); - auto num_elements = BytesBigToUInt64(columns_blob, shift_idx, kInt32Type); - *num_bytes = sizeof(T) * num_elements; - - // Parse integer array - uint64_t i_source = shift_idx + kBytesOfColumnLen + (num_elements + kNumDataOfByte - 1) / kNumDataOfByte; - auto array_data = std::make_unique(num_elements); - - for (uint64_t i = 0; i < num_elements; i++) { - uint8_t iBitMap = columns_blob[shift_idx + kBytesOfColumnLen + i / kNumDataOfByte]; - uint64_t i_type = (iBitMap >> ((kNumDataOfByte - 1 - (i % kNumDataOfByte)) * kDataTypeBits)) & kDataTypeBitMask; - auto mr_int_type = static_cast(i_type); - int64_t i64 = BytesLittleToMinIntType(columns_blob, i_source, mr_int_type); - i_source += (kUnsignedOne << i_type); - array_data[i] = static_cast(i64); - } - - auto data = reinterpret_cast(array_data.get()); - *data_ptr = std::make_unique(*num_bytes); - // field is none. for example: numpy is null - if (*num_bytes == 0) { - return Status::OK(); - } - CHECK_FAIL_RETURN_UNEXPECTED(memcpy_s(data_ptr->get(), *num_bytes, data, *num_bytes) == 0, - "[Internal ERROR] Failed to call securec func [memcpy_s]"); - return Status::OK(); -} - -uint64_t ShardColumn::BytesBigToUInt64(const std::vector &bytes_array, const uint64_t &pos, - const IntegerType &i_type) { - uint64_t result = 0; - for (uint64_t i = 0; i < (kUnsignedOne << static_cast(i_type)); i++) { - result = (result << kBitsOfByte) + bytes_array[pos + i]; - } - return result; -} - -std::vector ShardColumn::UIntToBytesBig(uint64_t value, const IntegerType &i_type) { - uint64_t n_bytes = kUnsignedOne << static_cast(i_type); - std::vector result(n_bytes, 0); - for (uint64_t i = 0; i < n_bytes; i++) { - result[n_bytes - 1 - i] = value & std::numeric_limits::max(); - value >>= kBitsOfByte; - } - return result; -} - -std::vector ShardColumn::UIntToBytesLittle(uint64_t value, const IntegerType &i_type) { - uint64_t n_bytes = kUnsignedOne << static_cast(i_type); - std::vector result(n_bytes, 0); - for (uint64_t i = 0; i < n_bytes; i++) { - result[i] = value & std::numeric_limits::max(); - value >>= kBitsOfByte; - } - return result; -} - -int64_t ShardColumn::BytesLittleToMinIntType(const std::vector &bytes_array, const uint64_t &pos, - const IntegerType &src_i_type, IntegerType *dst_i_type) { - uint64_t u_temp = 0; - for (uint64_t i = 0; i < (kUnsignedOne << static_cast(src_i_type)); i++) { - u_temp = (u_temp << kBitsOfByte) + - bytes_array[pos + (kUnsignedOne << static_cast(src_i_type)) - kUnsignedOne - i]; - } - - int64_t i_out; - switch (src_i_type) { - case kInt8Type: { - i_out = (int8_t)(u_temp & std::numeric_limits::max()); - break; - } - case kInt16Type: { - i_out = (int16_t)(u_temp & std::numeric_limits::max()); - break; - } - case kInt32Type: { - i_out = (int32_t)(u_temp & std::numeric_limits::max()); - break; - } - case kInt64Type: { - i_out = (int64_t)(u_temp & std::numeric_limits::max()); - break; - } - default: { - i_out = 0; - } - } - - if (!dst_i_type) { - return i_out; - } - - if (i_out >= static_cast(std::numeric_limits::min()) && - i_out <= static_cast(std::numeric_limits::max())) { - *dst_i_type = kInt8Type; - } else if (i_out >= static_cast(std::numeric_limits::min()) && - i_out <= static_cast(std::numeric_limits::max())) { - *dst_i_type = kInt16Type; - } else if (i_out >= static_cast(std::numeric_limits::min()) && - i_out <= static_cast(std::numeric_limits::max())) { - *dst_i_type = kInt32Type; - } else { - *dst_i_type = kInt64Type; - } - return i_out; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 9814edce78e73c6489223f519bbe16599a983a5f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:53:53 +0800 Subject: [PATCH 108/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc' --- .../include/meta/shard_distributed_sample.cc | 80 ------------------- 1 file changed, 80 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc deleted file mode 100644 index cb1d06efdb2..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_distributed_sample.cc +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2020-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_distributed_sample.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; - -namespace mindspore { -namespace mindrecord { -ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, int64_t no_of_padded_samples, bool shuffle, - uint32_t seed, int64_t no_of_samples, int64_t offset) - : ShardSample(1, num_shards, shard_id, no_of_samples, offset), - shuffle_(shuffle), - no_of_padded_samples_(no_of_padded_samples), - first_epoch_(true) { - shuffle_op_ = std::make_shared(seed, kShuffleSample); -} - -ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, bool shuffle, uint32_t seed, - int64_t no_of_samples, int64_t offset) - : ShardDistributedSample(num_shards, shard_id, 0, shuffle, seed, no_of_samples, offset) {} - -int64_t ShardDistributedSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (no_of_padded_samples_ <= 0) { - int64_t res = 0; - if (dataset_size % denominator_ == 0) { - res = dataset_size / denominator_ * numerator_; - } else { - res = dataset_size / denominator_ * numerator_ + 1; - } - return no_of_samples_ == 0 ? res : std::min(no_of_samples_, res); - } else { - auto padded_size = dataset_size + no_of_padded_samples_; - if (padded_size % denominator_ == 0) { - return padded_size / denominator_ * numerator_; - } else { - return -1; - } - } - return 0; -} - -Status ShardDistributedSample::PreExecute(ShardTaskList &tasks) { - auto total_no = tasks.Size(); - if (no_of_padded_samples_ > 0 && first_epoch_) { - CHECK_FAIL_RETURN_UNEXPECTED(total_no % denominator_ == 0, - "Invalid data, the size of dataset and padded samples: " + std::to_string(total_no) + - " can not be divisible by the value of 'num_shards': " + - std::to_string(denominator_) + ".\n Please adjust the value of 'num_padded'."); - } - if (first_epoch_) { - first_epoch_ = false; - task_ = tasks; - } else { - tasks = task_; - } - if (shuffle_ == true) { - shuffle_op_->SetShardSampleCount(GetShardSampleCount()); - shuffle_op_->UpdateShuffleMode(GetShuffleMode()); - RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); - } - return Status::OK(); -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 98182390236699f40a3781fbba18f2a7004dba6a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:54:14 +0800 Subject: [PATCH 109/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc' --- .../mindrecord/include/meta/shard_index.cc | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc deleted file mode 100644 index 73397b5bba0..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_index.cc +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_index.h" - -namespace mindspore { -namespace mindrecord { -// table name for index -const char TABLENAME[] = "index_table"; - -Index::Index() : database_name_(""), table_name_(TABLENAME) {} - -void Index::AddIndexField(const int64_t &schemaId, const std::string &field) { - fields_.emplace_back(pair(schemaId, field)); -} - -// Get attribute list -std::vector> Index::GetFields() { return fields_; } -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 4748a3c3bab77365bd2ae6c325176d55cf1c56e4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:54:32 +0800 Subject: [PATCH 110/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc' --- .../mindrecord/include/meta/shard_page.cc | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc deleted file mode 100644 index ba2292415f6..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_page.cc +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_page.h" -#include "pybind11/pybind11.h" - -namespace mindspore { -namespace mindrecord { -json Page::GetPage() const { - json str_page; - str_page["page_id"] = page_id_; - str_page["shard_id"] = shard_id_; - str_page["page_type"] = page_type_; - str_page["page_type_id"] = page_type_id_; - str_page["start_row_id"] = start_row_id_; - str_page["end_row_id"] = end_row_id_; - if (row_group_ids_.size() == 0) { - json row_groups = json({}); - row_groups["id"] = 0; - row_groups["offset"] = 0; - str_page["row_group_ids"].push_back(row_groups); - } else { - for (const auto &rg : row_group_ids_) { - json row_groups = json({}); - row_groups["id"] = rg.first; - row_groups["offset"] = rg.second; - str_page["row_group_ids"].push_back(row_groups); - } - } - str_page["page_size"] = page_size_; - return str_page; -} - -void Page::DeleteLastGroupId() { - if (!row_group_ids_.empty()) { - page_size_ = row_group_ids_.back().second; - row_group_ids_.pop_back(); - } -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 7140217d4e17918fc64ab122bdcc8190cfe8925a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:54:52 +0800 Subject: [PATCH 111/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc' --- .../include/meta/shard_pk_sample.cc | 47 ------------------- 1 file changed, 47 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc deleted file mode 100644 index 3974128fd5b..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_pk_sample.cc +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_pk_sample.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; - -namespace mindspore { -namespace mindrecord { -ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples) - : ShardCategory(category_field, num_elements, std::numeric_limits::max(), true), - shuffle_(false), - num_samples_(num_samples) {} - -ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, - int64_t num_samples) - : ShardCategory(category_field, num_elements, num_categories, true), shuffle_(false), num_samples_(num_samples) {} - -ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, - uint32_t seed, int64_t num_samples) - : ShardCategory(category_field, num_elements, num_categories, true), shuffle_(true), num_samples_(num_samples) { - shuffle_op_ = std::make_shared(seed, kShuffleSample); // do shuffle and replacement -} - -Status ShardPkSample::SufExecute(ShardTaskList &tasks) { - if (shuffle_ == true) { - RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); - } - return Status::OK(); -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 2a034be046716c7613e20e9bd087d2603a3d5d2c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:56:43 +0800 Subject: [PATCH 112/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc' --- .../mindrecord/include/meta/shard_sample.cc | 176 ------------------ 1 file changed, 176 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc deleted file mode 100644 index cbafe84cb8f..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sample.cc +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_sample.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; - -namespace mindspore { -namespace mindrecord { -ShardSample::ShardSample(int64_t n) - : numerator_(0), - denominator_(0), - partition_id_(0), - no_of_samples_(n), - indices_({}), - sampler_type_(kCustomTopNSampler), - offset_(-1) {} - -ShardSample::ShardSample(int64_t num, int64_t den) - : numerator_(num), - denominator_(den), - partition_id_(0), - no_of_samples_(0), - indices_({}), - sampler_type_(kCustomTopPercentSampler), - offset_(-1) {} - -ShardSample::ShardSample(int64_t num, int64_t den, int64_t par, int64_t no_of_samples, int64_t offset) - : numerator_(num), - denominator_(den), - partition_id_(par), - no_of_samples_(no_of_samples), - indices_({}), - sampler_type_(kCustomTopPercentSampler), - offset_(offset) {} - -ShardSample::ShardSample(const std::vector &indices) - : numerator_(0), - denominator_(0), - partition_id_(0), - no_of_samples_(0), - indices_(indices), - sampler_type_(kSubsetSampler) {} - -ShardSample::ShardSample(const std::vector &indices, uint32_t seed) : ShardSample(indices) { - sampler_type_ = kSubsetRandomSampler; - shuffle_op_ = std::make_shared(seed); -} - -int64_t ShardSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (sampler_type_ == kCustomTopNSampler) { - return no_of_samples_; - } - - if (sampler_type_ == kCustomTopPercentSampler) { - if (dataset_size % denominator_ == 0) { - return dataset_size / denominator_ * numerator_; - } else { - return dataset_size / denominator_ * numerator_ + 1; - } - } - if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { - return indices_.size(); - } - return 0; -} - -Status ShardSample::UpdateTasks(ShardTaskList &tasks, int64_t taking) { - if (tasks.permutation_.empty()) { - ShardTaskList new_tasks; - auto total_no = tasks.sample_ids_.size(); - CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0, - "[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no)); - if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { - for (int64_t i = 0; i < indices_.size(); ++i) { - int64_t index = ((indices_[i] % total_no) + total_no) % total_no; - new_tasks.AssignTask(tasks, index); // different mod result between c and python - } - } else { - int64_t count = 0; - if (nums_per_shard_.empty()) { - for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { - if (no_of_samples_ != 0 && count == no_of_samples_) break; - new_tasks.AssignTask(tasks, i % total_no); // rounding up. if overflow, go back to start - count++; - } - } else { - // Get samples within a specific range - int64_t i = partition_id_ - 1 >= 0 ? nums_per_shard_[partition_id_ - 1] : 0; - for (; i < nums_per_shard_[partition_id_]; i++) { - if (no_of_samples_ != 0 && count == no_of_samples_) break; - new_tasks.AssignTask(tasks, i % total_no); - count++; - } - } - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - } else { - ShardTaskList new_tasks; - int64_t total_no = tasks.permutation_.size(); - CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0, - "[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no)); - int64_t cnt = 0; - for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { - if (no_of_samples_ != 0 && cnt == no_of_samples_) break; - new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]); - cnt++; - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - } - return Status::OK(); -} - -Status ShardSample::Execute(ShardTaskList &tasks) { - if (offset_ != -1) { - int64_t old_v = 0; - int64_t num_rows_ = tasks.sample_ids_.size(); - for (int64_t x = 0; x < denominator_; x++) { - int64_t samples_per_buffer_ = (num_rows_ + offset_) / denominator_; - int64_t remainder = (num_rows_ + offset_) % denominator_; - if (x < remainder) samples_per_buffer_++; - if (x < offset_) samples_per_buffer_--; - old_v += samples_per_buffer_; - // nums_per_shard_ is used to save the current shard's ending index - nums_per_shard_.push_back(old_v); - } - } - int no_of_categories = static_cast(tasks.categories); - int64_t total_no = tasks.sample_ids_.size(); - int64_t taking = 0; - if (sampler_type_ == kCustomTopNSampler) { // non sharding case constructor #1 - no_of_samples_ = std::min(no_of_samples_, total_no); - taking = no_of_samples_ - no_of_samples_ % no_of_categories; - } else if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { - CHECK_FAIL_RETURN_UNEXPECTED(static_cast(indices_.size()) <= total_no, - "Invalid input, indices size: " + std::to_string(indices_.size()) + - " should be less than or equal to database size: " + std::to_string(total_no) + "."); - } else { // constructor TopPercent - if (numerator_ > 0 && denominator_ > 0 && numerator_ <= denominator_) { - if (numerator_ == 1 && denominator_ > 1) { // sharding - taking = (total_no + denominator_ - 1) / denominator_; - } else { // non sharding - taking = total_no * numerator_ / denominator_; - taking -= (taking % no_of_categories); - } - } else { - RETURN_STATUS_UNEXPECTED("[Internal ERROR] 'numerator_': " + std::to_string(numerator_) + - " should be positive and less than denominator_: " + std::to_string(denominator_) + "."); - } - } - return UpdateTasks(tasks, taking); -} - -Status ShardSample::SufExecute(ShardTaskList &tasks) { - if (sampler_type_ == kSubsetRandomSampler) { - RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); - } - return Status::OK(); -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 893d492d650dc5ad6fd7c2916c91ed1d1089c5e4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:57:05 +0800 Subject: [PATCH 113/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc' --- .../mindrecord/include/meta/shard_schema.cc | 163 ------------------ 1 file changed, 163 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc deleted file mode 100644 index d48d7694755..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_schema.cc +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_schema.h" -#include "utils/ms_utils.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; - -namespace mindspore { -namespace mindrecord { -std::shared_ptr Schema::Build(std::string desc, const json &schema) { - // validate check - if (!Validate(schema)) { - return nullptr; - } - - std::vector blob_fields = PopulateBlobFields(schema); - Schema object_schema; - object_schema.desc_ = std::move(desc); - object_schema.blob_fields_ = std::move(blob_fields); - object_schema.schema_ = schema; - object_schema.schema_id_ = -1; - return std::make_shared(object_schema); -} - -std::string Schema::GetDesc() const { return desc_; } - -json Schema::GetSchema() const { - json str_schema; - str_schema["desc"] = desc_; - str_schema["schema"] = schema_; - str_schema["blob_fields"] = blob_fields_; - return str_schema; -} - -void Schema::SetSchemaID(int64_t id) { schema_id_ = id; } - -int64_t Schema::GetSchemaID() const { return schema_id_; } - -std::vector Schema::GetBlobFields() const { return blob_fields_; } - -std::vector Schema::PopulateBlobFields(json schema) { - std::vector blob_fields; - for (json::iterator it = schema.begin(); it != schema.end(); ++it) { - json it_value = it.value(); - if ((it_value.size() == kInt2 && it_value.find("shape") != it_value.end()) || it_value["type"] == "bytes") { - blob_fields.emplace_back(it.key()); - } - } - return blob_fields; -} - -bool Schema::ValidateNumberShape(const json &it_value) { - if (it_value.find("shape") == it_value.end()) { - MS_LOG(ERROR) << "Invalid schema, 'shape' object can not found in " << it_value.dump() - << ". Please check the input schema."; - return false; - } - - auto shape = it_value["shape"]; - if (!shape.is_array()) { - MS_LOG(ERROR) << "Invalid schema, the value of 'shape' should be list format but got: " << it_value["shape"] - << ". Please check the input schema."; - return false; - } - - int num_negtive_one = 0; - for (const auto &i : shape) { - if (i == 0 || i < -1) { - MS_LOG(ERROR) << "Invalid schema, the element of 'shape' value should be -1 or greater than 0 but got: " << i - << ". Please check the input schema."; - return false; - } - if (i == -1) { - num_negtive_one++; - } - } - - if (num_negtive_one > 1) { - MS_LOG(ERROR) << "Invalid schema, only 1 variable dimension(-1) allowed in 'shape' value but got: " - << it_value["shape"] << ". Please check the input schema."; - return false; - } - - return true; -} - -bool Schema::Validate(json schema) { - if (schema.empty()) { - MS_LOG(ERROR) << "Invalid schema, schema is empty. Please check the input schema."; - return false; - } - - for (json::iterator it = schema.begin(); it != schema.end(); ++it) { - // make sure schema key name must be composed of '0-9' or 'a-z' or 'A-Z' or '_' - if (!ValidateFieldName(it.key())) { - MS_LOG(ERROR) << "Invalid schema, field name: " << it.key() - << "is not composed of '0-9' or 'a-z' or 'A-Z' or '_'. Please rename the field name in schema."; - return false; - } - - json it_value = it.value(); - if (it_value.find("type") == it_value.end()) { - MS_LOG(ERROR) << "Invalid schema, 'type' object can not found in field " << it_value.dump() - << ". Please add the 'type' object for field in schema."; - return false; - } - - if (kFieldTypeSet.find(it_value["type"]) == kFieldTypeSet.end()) { - MS_LOG(ERROR) << "Invalid schema, the value of 'type': " << it_value["type"] - << " is not supported.\nPlease modify the value of 'type' to 'int32', 'int64', 'float32', " - "'float64', 'string', 'bytes' in schema."; - return false; - } - - if (it_value.size() == kInt1) { - continue; - } - - if (it_value["type"] == "bytes" || it_value["type"] == "string") { - MS_LOG(ERROR) - << "Invalid schema, no other field can be added when the value of 'type' is 'string' or 'types' but got: " - << it_value.dump() << ". Please remove other fields in schema."; - return false; - } - - if (it_value.size() != kInt2) { - MS_LOG(ERROR) << "Invalid schema, the fields should be 'type' or 'type' and 'shape' but got: " << it_value.dump() - << ". Please check the schema."; - return false; - } - - if (!ValidateNumberShape(it_value)) { - return false; - } - } - - return true; -} - -bool Schema::operator==(const mindrecord::Schema &b) const { - if (this->GetDesc() != b.GetDesc() || this->GetSchema() != b.GetSchema()) { - return false; - } - return true; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 32ebd95cd0f0aafe8483f8bbf9382a0cd67e12ce Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:57:31 +0800 Subject: [PATCH 114/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc' --- .../include/meta/shard_sequential_sample.cc | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc deleted file mode 100644 index 2de121e8c22..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_sequential_sample.cc +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2020-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_sequential_sample.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; - -namespace mindspore { -namespace mindrecord { -ShardSequentialSample::ShardSequentialSample(int64_t n, int64_t offset) - : ShardSample(n), offset_(offset), per_(0.0f), per_offset_(0.0f) {} - -ShardSequentialSample::ShardSequentialSample(float per, float per_offset) - : ShardSample(0), offset_(0), per_(per), per_offset_(per_offset) {} - -int64_t ShardSequentialSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) { - return dataset_size; - } - if (per_ > kEpsilon && per_ <= 1.0f) { - return dataset_size * kEpsilon; - } - return std::min(static_cast(no_of_samples_), dataset_size); -} - -Status ShardSequentialSample::Execute(ShardTaskList &tasks) { - int64_t taking; - int64_t total_no = static_cast(tasks.sample_ids_.size()); - if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) { - taking = total_no; - } else if (per_ > kEpsilon && per_ <= 1.0f) { - taking = total_no * kEpsilon; - } else { - taking = std::min(static_cast(no_of_samples_), total_no); - } - - if (tasks.permutation_.empty()) { - ShardTaskList new_tasks; - total_no = static_cast(tasks.Size()); - for (int64_t i = offset_; i < taking + offset_; ++i) { - new_tasks.AssignTask(tasks, i % total_no); - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - } else { // shuffled - ShardTaskList new_tasks; - total_no = static_cast(tasks.permutation_.size()); - for (int64_t i = offset_; i < taking + offset_; ++i) { - new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]); - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - } - return Status::OK(); -} - -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From cd4636b8914c86296f25dca0f1d76c0cd9bdfc40 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:57:51 +0800 Subject: [PATCH 115/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc' --- .../mindrecord/include/meta/shard_shuffle.cc | 203 ------------------ 1 file changed, 203 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc deleted file mode 100644 index 4dd331b3316..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_shuffle.cc +++ /dev/null @@ -1,203 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_shuffle.h" - -#include - -namespace mindspore { -namespace mindrecord { -ShardShuffle::ShardShuffle(uint32_t seed, ShuffleType shuffle_type) - : shuffle_seed_(seed), - no_of_samples_(0), - replacement_(false), - reshuffle_each_epoch_(true), - shuffle_type_(shuffle_type) {} - -ShardShuffle::ShardShuffle(uint32_t seed, int64_t no_of_samples, bool replacement, bool reshuffle_each_epoch, - ShuffleType shuffle_type) - : shuffle_seed_(seed), - no_of_samples_(no_of_samples), - replacement_(replacement), - reshuffle_each_epoch_(reshuffle_each_epoch), - shuffle_type_(shuffle_type) {} - -int64_t ShardShuffle::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (replacement_) { - return no_of_samples_ == 0 ? dataset_size : no_of_samples_; - } - return no_of_samples_ == 0 ? dataset_size : std::min(dataset_size, no_of_samples_); -} - -Status ShardShuffle::CategoryShuffle(ShardTaskList &tasks) { - int64_t individual_size = tasks.sample_ids_.size() / tasks.categories; - std::vector> new_permutations(tasks.categories, std::vector(individual_size)); - for (int64_t i = 0; i < tasks.categories; i++) { - for (int64_t j = 0; j < individual_size; j++) new_permutations[i][j] = j; - std::shuffle(new_permutations[i].begin(), new_permutations[i].end(), std::default_random_engine(shuffle_seed_)); - } - tasks.permutation_.clear(); - for (int64_t j = 0; j < individual_size; j++) { - for (int64_t i = 0; i < tasks.categories; i++) { - tasks.permutation_.push_back(new_permutations[i][j] * tasks.categories + i); - } - } - - ShardTaskList new_tasks; - for (int64_t i = 0; i < individual_size; ++i) { - new_tasks.AssignTask(tasks, tasks.permutation_[i]); - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - - return Status::OK(); -} - -Status ShardShuffle::ShuffleFiles(ShardTaskList &tasks) { - if (no_of_samples_ == 0) { - no_of_samples_ = tasks.Size(); - } - CHECK_FAIL_RETURN_UNEXPECTED( - no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_)); - auto shard_sample_cout = GetShardSampleCount(); - - // shuffle the files index - std::vector shuffle_files; - for (int64_t i = 0; i < shard_sample_cout.size(); i++) { - shuffle_files.push_back(i); - } - std::shuffle(shuffle_files.begin(), shuffle_files.end(), std::default_random_engine(shuffle_seed_)); - - // reconstruct the permutation between files - // -- before -- - // file1: [0, 1, 2] - // file2: [3, 4, 5, 6] - // file3: [7, 8] - // file4: [9, 10] - // files: [file1, file2, file3, file4] - // permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - // -- after -- - // files: [file4, file1, file3, file2] - // permutation : [9, 10, 0, 1, 2, 7, 8, 3, 4, 5, 6] - auto original_permutation = tasks.permutation_; - int64_t whole_index = 0; - for (int64_t i = 0; i < shuffle_files.size(); i++) { - int64_t start_index = 0; - int64_t current_size = 0; - if (shuffle_files[i] == 0) { - start_index = 0; - current_size = shard_sample_cout[shuffle_files[i]]; - } else { - start_index = shard_sample_cout[shuffle_files[i] - 1]; - current_size = shard_sample_cout[shuffle_files[i]] - start_index; - } - std::copy(original_permutation.begin() + start_index, original_permutation.begin() + start_index + current_size, - tasks.permutation_.begin() + whole_index); - whole_index += current_size; - } - - auto total_no = tasks.Size(); - int64_t samples_to_assign = - (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); - ShardTaskList new_tasks; - for (int64_t i = 0; i < samples_to_assign; ++i) { - new_tasks.AssignTask(tasks, tasks.permutation_[i]); - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - return Status::OK(); -} - -Status ShardShuffle::ShuffleInfile(ShardTaskList &tasks) { - if (no_of_samples_ == 0) { - no_of_samples_ = tasks.Size(); - } - CHECK_FAIL_RETURN_UNEXPECTED( - no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_)); - // reconstruct the permutation in file - // -- before -- - // file1: [0, 1, 2] - // file2: [3, 4, 5, 6] - // file3: [7, 8] - // file4: [9, 10] - // files: [file1, file2, file3, file4] - // permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - // -- after -- - // permutation: [2, 0, 1, 4, 6, 3, 5, 8, 7, 9, 10] - auto shard_sample_cout = GetShardSampleCount(); - int64_t start_index = 0; - for (int64_t i = 0; i < shard_sample_cout.size(); i++) { - auto current_size = shard_sample_cout[i] - start_index; - std::shuffle(tasks.permutation_.begin() + start_index, tasks.permutation_.begin() + start_index + current_size, - std::default_random_engine(shuffle_seed_)); - start_index = shard_sample_cout[i]; - } - auto total_no = tasks.Size(); - ShardTaskList new_tasks; - int64_t samples_to_assign = - (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); - for (int64_t i = 0; i < samples_to_assign; ++i) { - new_tasks.AssignTask(tasks, tasks.permutation_[i]); - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - return Status::OK(); -} - -Status ShardShuffle::Execute(ShardTaskList &tasks) { - if (reshuffle_each_epoch_) { - shuffle_seed_++; - } - CHECK_FAIL_RETURN_UNEXPECTED(tasks.categories >= 1, - "[Internal ERROR] task categories should be greater than or equal to 1 but got: " + - std::to_string(tasks.categories)); - if (shuffle_type_ == kShuffleSample) { // shuffle each sample - if (tasks.permutation_.empty() == true) { - tasks.MakePerm(); - } - if (GetShuffleMode() == dataset::ShuffleMode::kGlobal) { - if (replacement_ == true) { - ShardTaskList new_tasks; - if (no_of_samples_ == 0) { - no_of_samples_ = tasks.sample_ids_.size(); - } - CHECK_FAIL_RETURN_UNEXPECTED(no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + - std::to_string(no_of_samples_)); - for (uint32_t i = 0; i < no_of_samples_; ++i) { - new_tasks.AssignTask(tasks, tasks.GetRandomTaskID()); - } - - ShardTaskList::TaskListSwap(tasks, new_tasks); - } else { - std::shuffle(tasks.permutation_.begin(), tasks.permutation_.end(), std::default_random_engine(shuffle_seed_)); - auto total_no = tasks.Size(); - ShardTaskList new_tasks; - int64_t samples_to_assign = - (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); - for (int64_t i = 0; i < samples_to_assign; ++i) { - new_tasks.AssignTask(tasks, tasks.permutation_[i]); - } - ShardTaskList::TaskListSwap(tasks, new_tasks); - } - } else if (GetShuffleMode() == dataset::ShuffleMode::kInfile) { - RETURN_IF_NOT_OK(ShuffleInfile(tasks)); - } else if (GetShuffleMode() == dataset::ShuffleMode::kFiles) { - RETURN_IF_NOT_OK(ShuffleFiles(tasks)); - } - } else { // shuffle unit like: (a1, b1, c1),(a2, b2, c2),..., (an, bn, cn) - return this->CategoryShuffle(tasks); - } - return Status::OK(); -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From c4c57ce21d45976ff34fd8414caa4592f7a6be2f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:58:11 +0800 Subject: [PATCH 116/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc' --- .../include/meta/shard_statistics.cc | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc deleted file mode 100644 index 07bcad01702..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_statistics.cc +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_statistics.h" -#include "pybind11/pybind11.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::ERROR; - -namespace mindspore { -namespace mindrecord { -std::shared_ptr Statistics::Build(std::string desc, const json &statistics) { - // validate check - if (!Validate(statistics)) { - return nullptr; - } - Statistics object_statistics; - object_statistics.desc_ = std::move(desc); - object_statistics.statistics_ = statistics; - object_statistics.statistics_id_ = -1; - return std::make_shared(object_statistics); -} - -std::string Statistics::GetDesc() const { return desc_; } - -json Statistics::GetStatistics() const { - json str_statistics; - str_statistics["desc"] = desc_; - str_statistics["statistics"] = statistics_; - return str_statistics; -} - -void Statistics::SetStatisticsID(int64_t id) { statistics_id_ = id; } - -int64_t Statistics::GetStatisticsID() const { return statistics_id_; } - -bool Statistics::Validate(const json &statistics) { - if (statistics.size() != kInt1) { - MS_LOG(ERROR) << "Invalid data, 'statistics' is empty."; - return false; - } - if (statistics.find("level") == statistics.end()) { - MS_LOG(ERROR) << "Invalid data, 'level' object can not found in statistic"; - return false; - } - return LevelRecursive(statistics["level"]); -} - -bool Statistics::LevelRecursive(json level) { - bool ini = true; - for (json::iterator it = level.begin(); it != level.end(); ++it) { - json a = it.value(); - if (a.size() == kInt2) { - if ((a.find("key") == a.end()) || (a.find("count") == a.end())) { - MS_LOG(ERROR) << "Invalid data, the node field is 2, but 'key'/'count' object does not existed"; - return false; - } - } else if (a.size() == kInt3) { - if ((a.find("key") == a.end()) || (a.find("count") == a.end()) || a.find("level") == a.end()) { - MS_LOG(ERROR) << "Invalid data, the node field is 3, but 'key'/'count'/'level' object does not existed"; - return false; - } else { - ini = LevelRecursive(a.at("level")); - } - } else { - MS_LOG(ERROR) << "Invalid data, the node field is not equal to 2 or 3"; - return false; - } - } - return ini; -} - -bool Statistics::operator==(const Statistics &b) const { - if (this->GetStatistics() != b.GetStatistics()) { - return false; - } - return true; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 89a3eaf60e7daf2747c3ae5200ad67368bf08962 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 21:59:49 +0800 Subject: [PATCH 117/160] ADD file via upload --- .../mindrecord/include/meta/shard_category.cc | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc new file mode 100644 index 00000000000..87194633898 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc @@ -0,0 +1,62 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_category.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardCategory空间中ShardCategory函数,输入三种参数调用指定函数 +//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 +ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, + bool replacement) + : categories_(categories), + category_field_(""), + num_elements_(num_elements), + num_categories_(0), + replacement_(replacement) {} +//引用ShardCategory空间中ShardCategory函数,输入四种参数调用指定函数 +//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 +ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, + bool replacement) + : categories_({}), + category_field_(category_field), + num_elements_(num_elements), + num_categories_(num_categories), + replacement_(replacement) {} + +Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); }//在ShardCategory空间中创建Status型Execute函数,返回值为Status空间中的OK函数的返回值 +//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为0或-1 +//判断dataset_size的值 +//若dataset_size的值为0,则返回dataset_size本身 +//若dataset_size的值大于0,则继续判断num_classes、num_categories_、num_elements_的值是否大于0。若均大于0,则修改num_classes的赋值,赋值为num_categories_和num_classes的最小值 +//继续判断num_classes是否为0,若为0则返回0。不为0则判断num_elements_是否大于int64_t类型最大值与num_classes的商,若是则返回-1。若均不符合则返回num_classes和num_elements_的乘积 +//若均不符合,则返回0 +int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (dataset_size == 0) return dataset_size; + if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { + num_classes = std::min(num_categories_, num_classes); + if (num_classes == 0) { + return 0; + } + if (num_elements_ > std::numeric_limits::max() / num_classes) { + return -1; + } + return num_classes * num_elements_; + } + return 0; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From a9d340d1863f3be38266873a0380d06077494131 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:00:37 +0800 Subject: [PATCH 118/160] ADD file via upload --- .../ccsrc/minddata/mindrecord/meta/int | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/int diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/int b/mindspore2022/ccsrc/minddata/mindrecord/meta/int new file mode 100644 index 00000000000..87194633898 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/int @@ -0,0 +1,62 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_category.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardCategory空间中ShardCategory函数,输入三种参数调用指定函数 +//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 +ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, + bool replacement) + : categories_(categories), + category_field_(""), + num_elements_(num_elements), + num_categories_(0), + replacement_(replacement) {} +//引用ShardCategory空间中ShardCategory函数,输入四种参数调用指定函数 +//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 +ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, + bool replacement) + : categories_({}), + category_field_(category_field), + num_elements_(num_elements), + num_categories_(num_categories), + replacement_(replacement) {} + +Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); }//在ShardCategory空间中创建Status型Execute函数,返回值为Status空间中的OK函数的返回值 +//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为0或-1 +//判断dataset_size的值 +//若dataset_size的值为0,则返回dataset_size本身 +//若dataset_size的值大于0,则继续判断num_classes、num_categories_、num_elements_的值是否大于0。若均大于0,则修改num_classes的赋值,赋值为num_categories_和num_classes的最小值 +//继续判断num_classes是否为0,若为0则返回0。不为0则判断num_elements_是否大于int64_t类型最大值与num_classes的商,若是则返回-1。若均不符合则返回num_classes和num_elements_的乘积 +//若均不符合,则返回0 +int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (dataset_size == 0) return dataset_size; + if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { + num_classes = std::min(num_categories_, num_classes); + if (num_classes == 0) { + return 0; + } + if (num_elements_ > std::numeric_limits::max() / num_classes) { + return -1; + } + return num_classes * num_elements_; + } + return 0; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 0235d39f67e30c77209434ad99034448715dbcad Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:01:00 +0800 Subject: [PATCH 119/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc' --- .../mindrecord/include/meta/shard_category.cc | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc deleted file mode 100644 index 87194633898..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_category.cc +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_category.h"//按照路径寻找以下文件，导入到本文件 - -namespace mindspore {//创建名为mindspore的空间 -namespace mindrecord {//创建名为mindrecord的空间 -//引用ShardCategory空间中ShardCategory函数,输入三种参数调用指定函数 -//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 -ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, - bool replacement) - : categories_(categories), - category_field_(""), - num_elements_(num_elements), - num_categories_(0), - replacement_(replacement) {} -//引用ShardCategory空间中ShardCategory函数,输入四种参数调用指定函数 -//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 -ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, - bool replacement) - : categories_({}), - category_field_(category_field), - num_elements_(num_elements), - num_categories_(num_categories), - replacement_(replacement) {} - -Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); }//在ShardCategory空间中创建Status型Execute函数,返回值为Status空间中的OK函数的返回值 -//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为0或-1 -//判断dataset_size的值 -//若dataset_size的值为0,则返回dataset_size本身 -//若dataset_size的值大于0,则继续判断num_classes、num_categories_、num_elements_的值是否大于0。若均大于0,则修改num_classes的赋值,赋值为num_categories_和num_classes的最小值 -//继续判断num_classes是否为0,若为0则返回0。不为0则判断num_elements_是否大于int64_t类型最大值与num_classes的商,若是则返回-1。若均不符合则返回num_classes和num_elements_的乘积 -//若均不符合,则返回0 -int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (dataset_size == 0) return dataset_size; - if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { - num_classes = std::min(num_categories_, num_classes); - if (num_classes == 0) { - return 0; - } - if (num_elements_ > std::numeric_limits::max() / num_classes) { - return -1; - } - return num_classes * num_elements_; - } - return 0; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 1b97d3e79ea4c73096a9d36ad8e939833c2d77ba Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:01:20 +0800 Subject: [PATCH 120/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/else' --- .../ccsrc/minddata/mindrecord/include/else | 549 ------------------ 1 file changed, 549 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/else diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/else b/mindspore2022/ccsrc/minddata/mindrecord/include/else deleted file mode 100644 index eda89bd3759..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/else +++ /dev/null @@ -1,549 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "include/transform/graph_ir/util.h" - -#include -#include - -#include "securec/include/securec.h" -#include "include/common/utils/convert_utils.h" -#include "include/common/utils/utils.h" - -namespace mindspore { -namespace transform { -using std::make_shared; -using std::shared_ptr; -using std::string; -using std::vector; - -const size_t kErrorSize = 0; -//在TransformUtil区域中创建名为ConvertIntToList的函数，返回值类型为vector，用于检测size的长度。 -//若长度小于等于零则弹出MS_LOG警告日志 "size <= 0"并直接返回list -//若长度大于零则用list对data中的内容进行push_back操作 -vector TransformUtil::ConvertIntToList(int64_t data, int size) { - vector list{}; - if (size <= 0) { - MS_LOG(WARNING) << "size <= 0"; - return list; - } - for (int i = 0; i < size; ++i) { - list.push_back(data); - } - return list; -} - -static std::map datatype_trans_map = { - {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, - {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, - {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, - {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, - {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, - {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; -//在TransformUtil区域中创建名为ConvertDataType的函数，返回值类型为自定义GeDataType -//输出MS_LOG调试日志"Convert me data type: "[*此处为对type进行TypeIdLabel(type)的结果]ge data type" -//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 -//若不一致，则返回datatype_trans_map[type] -//若一致则返回DT_UNDEFINED -GeDataType TransformUtil::ConvertDataType(const MeDataType &type) { - MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; - if (datatype_trans_map.find(type) != datatype_trans_map.end()) { - return datatype_trans_map[type]; - } else { - return GeDataType::DT_UNDEFINED; - } -} - -static std::map datatype_size_map = { - {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float - {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, - {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, - {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, - {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, - {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; -//在TransformUtil区域中创建名为GetDataTypeSize的函数，返回值类型为自定义size_t -//用于检测测变量type中datatype_size_map.find(type) 的结果是否与datatype_size_map.end()一致 -//若不一致，则返回datatype_trans_map[type] -//若一致，则输出错误日志 "Illegal tensor data type!"并返回kErrorSize -size_t TransformUtil::GetDataTypeSize(const MeDataType &type) { - if (datatype_size_map.find(type) != datatype_size_map.end()) { - return datatype_size_map[type]; - } else { - MS_LOG(ERROR) << "Illegal tensor data type!"; - return kErrorSize; - } -} -//在TransformUtil区域中创建名为:ConvertFormat的函数，返回值类型为自定义GeFormat -//判断format与 kOpFormat_NCHW等kOpFormat_*是否相等，若相等，则返回 kOpFormat_NCHW等kOpFormat_* -//若与一系列kOpFormat_*都不相等，则输出错误日志 "Illegal tensor data format: ("[*此处为format变量]"). Use ND format instead." -//并返回FORMAT_ND -GeFormat TransformUtil::ConvertFormat(const string &format) { - if (format == kOpFormat_NCHW) { - return GeFormat::FORMAT_NCHW; - } else if (format == kOpFormat_NDHWC) { - return GeFormat::FORMAT_NDHWC; - } else if (format == kOpFormat_NCDHW) { - return GeFormat::FORMAT_NCDHW; - } else if (format == kOpFormat_DHWNC) { - return GeFormat::FORMAT_DHWNC; - } else if (format == kOpFormat_DHWCN) { - return GeFormat::FORMAT_DHWCN; - } else if (format == kOpFormat_NC1HWC0) { - return GeFormat::FORMAT_NC1HWC0; - } else if (format == kOpFormat_NHWC) { - return GeFormat::FORMAT_NHWC; - } else if (format == kOpFormat_HWCN) { - return GeFormat::FORMAT_HWCN; - } else if (format == kOpFormat_ND) { - return GeFormat::FORMAT_ND; - } else { - MS_LOG(ERROR) << "Illegal tensor data format: (" << format << "). Use ND format instead."; - return GeFormat::FORMAT_ND; - } -} -//创建名为IntegerCastFunc的函数，返回值类型为自定义static int64_t -//返回执行static_cast()操作后的size_t的temp -static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } - -//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr TransformUtil::GetGeTensorDesc(const ShapeVector &me_shape, const MeDataType &me_type, - const std::string &format) { - // convert me shape to ge shape - //将me_shape转化为ge_shape - //定义ge_shape - std::vector ge_shape; - //若me_shape.size()=1，则对其第[0]位进行push_back操作 - //否则，则根据me_shape的size属性设置ge_shape的size属性 - if (me_shape.size() == 1) { - ge_shape.push_back(static_cast(me_shape[0])); - } else { - ge_shape.resize(me_shape.size()); - (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); - } - //若shape.GetDimNum()=0，则输出信息日志"The dims size of Ge tensor is zero" - GeShape shape(ge_shape); - if (shape.GetDimNum() == 0) { - MS_LOG(INFO) << "The dims size of Ge tensor is zero"; - } - // convert me format to ge format - //将me_format转化为ge_format - //若ge_format符合相应条件则输出信息日志 "Set ND data format" - GeFormat ge_format = ConvertFormat(format); - if (ge_format == GeFormat::FORMAT_ND) { - MS_LOG(INFO) << "Set ND data format"; - } - // convert me datatype to ge datatype - //将me_datatype转化为ge_datatype - //若data_type符合相应条件则输出错误日志"undefined data type :"[*此处为me_type] - //并返回 nullptr - GeDataType data_type = ConvertDataType(me_type); - if (data_type == GeDataType::DT_UNDEFINED) { - MS_LOG(ERROR) << "undefined data type :" << me_type; - return nullptr; - } - //进行make_shared操作，若desc变量为空则输出错误日志 "Create GeTensorDesc failed!" - //并返回 nullptr - //若不为空则输出信息日志"SetRealDimCnt is :"[*此处为 me_shape.size()] - //并对desc进行SetRealDimCnt操作 - //并返回desc - //这一步在检验转换是否成功并且转换后对象的size属性正确 - auto desc = std::make_shared(shape, ge_format, data_type); - if (desc == nullptr) { - MS_LOG(ERROR) << "Create GeTensorDesc failed!"; - return nullptr; - } - MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); - desc->SetRealDimCnt(SizeToInt(me_shape.size())); - return desc; -} - -// if failed, return empty vector. -//如果失败了，则返回空的vector -//在TransformUtil区域中创建名为GetGeTensorDesc的函数，返回值类型为自定义std::shared_ptr -std::vector TransformUtil::ConvertInputTensors(const std::vector &me_tensors, - const std::string &format) { - //定义ge_tensors - std::vector ge_tensors; - //在index小于me_tensors.size的范围内递增循环 - //输出消息日志 "me_tensor "[*此处为index(tensors的位数)]" 's data size is: "[*此处为index对应位的tensors的size] - for (size_t index = 0; index < me_tensors.size(); index++) { - MS_EXCEPTION_IF_NULL(me_tensors[index]); - MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); - //定义变量shape为index对应位tensors的shape - //将shape一个个写入shape_str中 - auto shape = me_tensors[index]->shape(); - std::string shape_str; - for (size_t i = 0; i < shape.size(); i++) { - shape_str += std::to_string(shape[i]); - shape_str += " "; - } - //输出消息日志 "me_tensor "[*此处为index(位数)]" 's shape is:"[*此处为index对应位的tensors的shape] - //输出消息日志"me_tensor "[*此处为index(位数)]" 's type is: "[*此处为index对应位的tensors的type] - MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; - MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); - //定义ge_tensor_ptr - auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); - //如果ge_tensor_ptr为空，则执行对应操作 - //否则，输出错误日志 "Convert me_tensor "[*此处为index(位数)]" to Ge Tensor failed!" - //清空ge_tensors并返回 - //这一步为检验此前过程是否成功 - if (ge_tensor_ptr != nullptr) { - ge_tensors.emplace_back(ge_tensor_ptr); - } else { - MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; - ge_tensors.clear(); - return ge_tensors; - } - } - return ge_tensors; -} - -GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr &tensor, const std::string &format) { - // get tensor data type size - //获取变量tensors的data_type的size属性 - //检验tensor是否为空，并输出相应消息 - MS_EXCEPTION_IF_NULL(tensor); - //获取变量tensors的data_type的size属性 - size_t type_size = GetDataTypeSize(tensor->data_type()); - //判断其是否为空，若为空则输出错误日志"The Me Tensor data type size is wrong, type size is: "[*此处为type_size] - if (type_size == kErrorSize) { - MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; - return nullptr; - } - size_t elements_num = IntToSize(tensor->ElementsNum()); - - // get tensor buff size - //获取变量tensor的data_buff_size属性 - size_t data_buff_size = elements_num * type_size; - //检验其是否为空，若为空，则输出消息日志"The Me Tensor data buff size is 0." - if (data_buff_size == 0) { - MS_LOG(INFO) << "The Me Tensor data buff size is 0."; - } - // create ge tensor - //创建ge_tensor - //将tensors的属性写入到desc中 - auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); - //检验其是否为空，若为空，则输出错误日志 "Failed to get Tensor Desc" - //这一步主要检验获取data_type等情况 - if (desc == nullptr) { - MS_LOG(ERROR) << "Failed to get Tensor Desc"; - return nullptr; - } - //将desc和tensors有关属性存入tensors_ptr中 - GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); - //检验其是否为空，若不为空，则输出消息日志"Convert Me Tensor to Ge Tensor success!" - //此处为检验上述过程是否成功 - if (tensor_ptr != nullptr) { - MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; - } - return tensor_ptr; -} -//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector -std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors, - const std::vector &request_dims) { - //定义outputs - std::vector outputs; - //在index小于ge_tensors.size的范围内递增循环 - //定义me_tensor_ptr为空 - for (size_t index = 0; index < ge_tensors.size(); index++) { - MeTensorPtr me_tensor_ptr = nullptr; - //判断index和request_dims.size的大小 - //若后者较大，则将index对应位置的ge_tensors和request_dims存入me_tensor_ptr - //反之则定义empty_shape并将其与index对应位置的ge_tensors存入me_tensor_ptr - if (index < request_dims.size()) { - me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); - } else { - ShapeVector empty_shape; - me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); - } - //判断me_tensor_ptr是否为空 - //若不为空，则输出me_tensor_ptr相应内容 - //若为空，则出书错误日志"Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs - //此处为检验上述判断大小部分的功能是否成功实现 - if (me_tensor_ptr != nullptr) { - outputs.emplace_back(me_tensor_ptr); - } else { - MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; - return outputs; - } - } - return outputs; -} -//在TransformUtil区域中创建名为ConvertGeTensors的函数，返回值类型为自定义std::vector -std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors) { - //定义outputs - std::vector outputs; - //在index小于ge_tensors.size的范围内递增循环 - //定义me_tensor_ptr，储存经ConvertGeTensor处理后的index位ge_tensors - for (size_t index = 0; index < ge_tensors.size(); index++) { - MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); - //若me_tensor_ptr不为空，则进行相应output处理 - //否则，输出错误日志 "Convert Ge Tensor "[*此处为index(位数)]" to Me Tensor failed!"并返回outputs - //主要检验ge_tensors与me_tensors的转化是否成功 - if (me_tensor_ptr != nullptr) { - outputs.emplace_back(me_tensor_ptr); - } else { - MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; - return outputs; - } - } - return outputs; -} -//在TransformUtil区域中创建名为:ConvertGeDataType的函数，返回值类型为自定义MeDataType -//此函数功能为检验type的类型，在type类型为DT_*的情况下返回对应的kNumberType* -//如果变量type不属于DT_*中的任何一种，则返回MeDataType::kTypeUnknown -MeDataType TransformUtil::ConvertGeDataType(const GeDataType &type) { - switch (type) { - case GeDataType::DT_FLOAT16: - return MeDataType::kNumberTypeFloat16; - case GeDataType::DT_FLOAT: - return MeDataType::kNumberTypeFloat32; - case GeDataType::DT_DOUBLE: - return MeDataType::kNumberTypeFloat64; - case GeDataType::DT_INT64: - return MeDataType::kNumberTypeInt64; - case GeDataType::DT_INT32: - return MeDataType::kNumberTypeInt32; - case GeDataType::DT_INT16: - return MeDataType::kNumberTypeInt16; - case GeDataType::DT_INT8: - return MeDataType::kNumberTypeInt8; - case GeDataType::DT_BOOL: - return MeDataType::kNumberTypeBool; - case GeDataType::DT_UINT8: - return MeDataType::kNumberTypeUInt8; - case GeDataType::DT_UINT16: - return MeDataType::kNumberTypeUInt16; - case GeDataType::DT_UINT32: - return MeDataType::kNumberTypeUInt32; - case GeDataType::DT_UINT64: - return MeDataType::kNumberTypeUInt64; - case GeDataType::DT_UNDEFINED: - case GeDataType::DT_DUAL_SUB_UINT8: - case GeDataType::DT_DUAL_SUB_INT8: - case GeDataType::DT_DUAL: - return MeDataType::kTypeUnknown; - default: - return MeDataType::kTypeUnknown; - } -} - -namespace { -//定义名为IsGeShapeCompatible的函数，返回值为bool -bool IsGeShapeCompatible(const GeShape &ge_shape, const ShapeVector &request_dims) { - //输出消息日志，说明当前Getensors的shape属性和我需要的shape属性 - MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); - MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); - - const int GE_DIMS = 4; - std::vector ge_dims = ge_shape.GetDims(); - //判断所需的size与当前size的大小，若前者较大，则输出错误日志 "Request shape's dims count greater than ge shape's" - //并返回false - if (request_dims.size() > ge_dims.size()) { - MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; - return false; - } - - // convert NHWC to NCHW 将NHWC转化为NCHW - //if括号中的内容为判断ge_tensors与re_quest的shape是否相容的条件 - //若符合条件则输出消息日志"Ge tensor shape and request shape is compatible"并返回true - if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && - (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { - MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; - return true; - } - //令i从0到request_dims.size范围内循环并判断ge_dims[i]与request_dims[i]是否相等 - //若不相等，则输出错误日志"Request shape's dims value not equal to ge shape's"并返回false - std::string::size_type i = 0; - for (; i < request_dims.size(); i++) { - if (ge_dims[i] != request_dims[i]) { - MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; - return false; - } - } - //判断i递增过程中对应的ge_dims[i]是否为1，并输出相应错误日志并返回false - for (; i < ge_dims.size(); i++) { - if (ge_dims[i] != 1) { - MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; - return false; - } - } - MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; - return true; -} -} // namespace - -//利用变量ge_dims实现变量me_dims向Geshape类型的转换 -GeShape TransformUtil::ConvertMeShape(const ShapeVector &me_dims) { - std::vector ge_dims; - (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); - return GeShape(ge_dims); -} -//利用变量me_dims实现变量ge_shape向ShapeVector类型的转换 -ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape) { - ShapeVector me_dims; - std::vector ge_dims = ge_shape.GetDims(); - (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); - return me_dims; -} -//在TransformUtil区域中创建名为ConvertGeShape的函数，返回值类型为自定义类型ShapeVector -ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape, const ShapeVector &request_dims) { - vector ret; - //判断ge_shape的对应属性是否为零，若为零则输出错误日志 "GeTensor's shape is scalar"并返回ret - if (ge_shape.GetDimNum() == 0) { - MS_LOG(DEBUG) << "GeTensor's shape is scalar"; - return ret; - } - //判断ge_shape是否可可兼容，若可兼容则用ret储存相应的值，否则输出相应错误日志并将ge_shape进行转换并用ret储存 - if (IsGeShapeCompatible(ge_shape, request_dims) == true) { - ret = request_dims; - } else { - MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; - ret = ConvertGeShape(ge_shape); - } - return ret; -} -//在TransformUtil区域中创建名为GenerateMeTensor的函数，返回值类型为自定义类型MeTensorPtr -MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const ShapeVector &me_dims, - const TypeId &me_type) { - MeTensor me_tensor(me_type, me_dims); - - // Get the writable data pointer of the tensor and cast it to its data type - //获取tensors的可写的数据指针并将其改写成其datatype - auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); - size_t me_data_size = static_cast(me_tensor.data().nbytes());//获取tensors的data size属性 - MS_EXCEPTION_IF_NULL(me_data_ptr);//检测me_data_ptr是否为空 - MS_EXCEPTION_IF_NULL(ge_tensor);//检测ge_tensor是否为空 - //判断me_data_size和ge_tensor的size的大小，若前者较小，则输出对应错误日志 - //"ME tensor data size["[*此处为me_data_size]" bytes] is less than GE tensor ["[*此处为ge_tensors的size]" bytes]" - //并返回nullptr - if (me_data_size < ge_tensor->GetSize()) { - MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" - << ge_tensor->GetSize() << " bytes]"; - return nullptr; - } - - // Copy or use the writable data pointer of the ME tensor - //复制或使用me_tensors的可写数据指针 - MS_EXCEPTION_IF_NULL(ge_tensor->GetData());//判断ge_tensors的data是否为空 - //判断ge_tensors的size属性是否为零，若为零，则输出相应错误日志并返回nullptr - if (ge_tensor->GetSize() == 0) { - MS_LOG(ERROR) << "GE tensor data size is zero!"; - return nullptr; - } - - // Use memcpy here, not memcpy_s, just because the size of ge_tensor may be bigger than 2GB - //因为ge_tensor的size可能大于2GB，所以在此处应用memcpy而不是memcpy_s - // which is the size limit of memcpy_s - //2GB为memcpy_s的极限容量 - (void)memcpy(me_data_ptr, ge_tensor->GetData(), ge_tensor->GetSize()); - - return make_shared(me_tensor); -} -//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr -MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr &ge_tensor) { - MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 - GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape - vector me_dims = ConvertGeShape(ge_shape); - - TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype - //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr - if (type_id == MeDataType::kTypeUnknown) { - MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " - << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - return nullptr; - } - return GenerateMeTensor(ge_tensor, me_dims, type_id); -} - -// if request_dims is empty, use ge tensor's shape, otherwise convert to request shape -//若 request_dim为空，则使用ge_tensors的shape，用其他的方式转化成要求的shape -//在TransformUtil区域中创建名为ConvertGeTensor的函数，返回值类型为自定义类型MeTensorPtr -MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const ShapeVector &request_dims) { - MS_EXCEPTION_IF_NULL(ge_tensor);//判断ge_tensors是否为空 - GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape();//定义变量ge_shape储存ge_tensors的shape - vector me_dims = ConvertGeShape(ge_shape, request_dims); - //输出信息日志 "GE tensor type is [*此处为ge_tensors的datatype]" - MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - // Create a tensor with wanted data type and shape - TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType());//定义变量type_id储存ge_tensors的datatype - //检验type_id中存入的内容是否支持转换，若不支持则输出错误日志并返回nullptr - if (type_id == MeDataType::kTypeUnknown) { - MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " - << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - return nullptr; - } - return GenerateMeTensor(ge_tensor, me_dims, type_id); -} -//在TransformUtil区域中创建名为PrintGeTensor的函数，返回值类型为std::string -std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { - std::string ret; - //检验变量ge_tensors是否为空，若为空则输出错误日志"Input ge tensor is nullptr"并返回ret - if (ge_tensor == nullptr) { - MS_LOG(ERROR) << "Input ge tensor is nullptr"; - return ret; - } - //ge_tensors不为空执行以下语句 - //输出信息日志"Ge Tensor data type is : [*此处为ge_tensors的data_type]" - //对于ge_trensors的data_type进行比对，若其符合GeDataType::DT_*等一系列的值，则令ret等于对应的值并结束语句 - //若其不符合这一系列的值，则输出错误日志"Unsupported to print type:"[*此处为ge_tensors的data type]" ge tensor";并结束语句 - MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); - switch (static_cast(ge_tensor->GetTensorDesc().GetDataType())) { - case GeDataType::DT_UINT32: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_FLOAT: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_INT32: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_DOUBLE: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_INT64: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_UINT64: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_INT16: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_UINT16: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_DUAL_SUB_INT8: - case GeDataType::DT_INT8: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_UINT8: - case GeDataType::DT_DUAL_SUB_UINT8: - ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); - break; - case GeDataType::DT_FLOAT16: - case GeDataType::DT_BOOL: - case GeDataType::DT_UNDEFINED: - case GeDataType::DT_DUAL: - default: - MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) - << " ge tensor"; - break; - } - return ret; -} -} // namespace transform -} // namespace mindspore -- 2.34.1 From fe1d2bc8ddfabb2a0345c625100838f4d1ade147 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:01:44 +0800 Subject: [PATCH 121/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc' --- .../include/meta/shard_task_list.cc | 146 ------------------ 1 file changed, 146 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc b/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc deleted file mode 100644 index 43a8e938794..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/meta/shard_task_list.cc +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/dataset/util/random.h" -#include "minddata/mindrecord/include/shard_task_list.h" -#include "utils/ms_utils.h" -#include "minddata/mindrecord/include/common/shard_utils.h" - -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::DEBUG; - -namespace mindspore { -namespace mindrecord { -ShardTaskList::ShardTaskList() : categories(1) {} - -ShardTaskList::ShardTaskList(const ShardTaskList &other) - : categories(other.categories), - permutation_(other.permutation_), - sample_ids_(other.sample_ids_), - task_list_(other.task_list_) {} - -ShardTaskList &ShardTaskList::operator=(const ShardTaskList &other) { - ShardTaskList tmp(other); - std::swap(categories, tmp.categories); - permutation_.swap(tmp.permutation_); - sample_ids_.swap(tmp.sample_ids_); - task_list_.swap(tmp.task_list_); - return *this; -} - -void ShardTaskList::InitSampleIds() { - // no-op if there already exists sample ids. Do not clobber previous list - if (sample_ids_.empty()) { - sample_ids_ = std::vector(task_list_.size()); - for (auto i = 0; i < task_list_.size(); i++) { - sample_ids_[i] = i; - } - } -} - -void ShardTaskList::MakePerm() { - int64_t perm_size = sample_ids_.size(); - permutation_ = std::vector(perm_size); - for (int64_t i = 0; i < perm_size; i++) { - permutation_[i] = i; - } -} - -// Swap the new_tasks with orig_tasks -void ShardTaskList::TaskListSwap(ShardTaskList &orig_tasks, ShardTaskList &new_tasks) { - // When swapping, if the orig_tasks contains fields that need to be preserved after the swap, then swapping with a - // new_tasks that does not have those fields will result in clobbering/losing the data after the swap. - // The task_list_ should not be lost/clobbered. - // This function can be called in the middle of mindrecord's epoch, when orig_tasks.task_list_ is still being - // used by mindrecord op's worker threads. So don't touch its task_list_ since this field should be preserved anyways. - - std::swap(orig_tasks.categories, new_tasks.categories); - std::swap(orig_tasks.permutation_, new_tasks.permutation_); - std::swap(orig_tasks.sample_ids_, new_tasks.sample_ids_); -} - -void ShardTaskList::PopBack() { task_list_.pop_back(); } - -int64_t ShardTaskList::Size() const { return static_cast(task_list_.size()); } - -int64_t ShardTaskList::SizeOfRows() const { - if (task_list_.size() == 0) return static_cast(0); - - // 1 task is 1 page - const size_t kBlobInfoIndex = 2; - auto sum_num_rows = [](int64_t x, ShardTask y) { return x + std::get(y)[0]; }; - int64_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows); - return nRows; -} - -ShardTask &ShardTaskList::GetTaskByID(int64_t id) { return task_list_[id]; } - -int64_t ShardTaskList::GetTaskSampleByID(int64_t id) { return sample_ids_[id]; } - -int64_t ShardTaskList::GetRandomTaskID() { - std::mt19937 gen = mindspore::dataset::GetRandomDevice(); - std::uniform_int_distribution<> dis(0, sample_ids_.size() - 1); - return dis(gen); -} - -ShardTask &ShardTaskList::GetRandomTask() { - std::mt19937 gen = mindspore::dataset::GetRandomDevice(); - std::uniform_int_distribution<> dis(0, task_list_.size() - 1); - return task_list_[dis(gen)]; -} - -ShardTaskList ShardTaskList::Combine(std::vector &category_tasks, bool replacement, int64_t num_elements, - int64_t num_samples) { - ShardTaskList res; - if (category_tasks.empty()) return res; - auto total_categories = category_tasks.size(); - res.categories = static_cast(total_categories); - if (replacement == false) { - auto minTasks = category_tasks[0].Size(); - for (int64_t i = 1; i < total_categories; i++) { - minTasks = std::min(minTasks, category_tasks[i].Size()); - } - int64_t count = 0; - for (int64_t task_no = 0; task_no < minTasks; task_no++) { - for (int64_t i = 0; i < total_categories; i++) { - if (num_samples != 0 && count == num_samples) break; - res.InsertTask(std::move(category_tasks[i].GetTaskByID(task_no))); - count++; - } - } - } else { - auto maxTasks = category_tasks[0].Size(); - for (int64_t i = 1; i < total_categories; i++) { - maxTasks = std::max(maxTasks, category_tasks[i].Size()); - } - if (num_elements != std::numeric_limits::max()) { - maxTasks = static_cast(num_elements); - } - int64_t count = 0; - for (int64_t i = 0; i < total_categories; i++) { - for (int64_t j = 0; j < maxTasks; j++) { - if (num_samples != 0 && count == num_samples) break; - res.InsertTask(category_tasks[i].GetRandomTask()); - count++; - } - } - } - - return res; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From 652332128c9223b92066d0d6fbbbec5f5b18e908 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:02:03 +0800 Subject: [PATCH 122/160] ADD file via upload --- .../mindrecord/meta/shard_category.cc | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_category.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_category.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_category.cc new file mode 100644 index 00000000000..87194633898 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_category.cc @@ -0,0 +1,62 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_category.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardCategory空间中ShardCategory函数,输入三种参数调用指定函数 +//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 +ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, + bool replacement) + : categories_(categories), + category_field_(""), + num_elements_(num_elements), + num_categories_(0), + replacement_(replacement) {} +//引用ShardCategory空间中ShardCategory函数,输入四种参数调用指定函数 +//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 +ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, + bool replacement) + : categories_({}), + category_field_(category_field), + num_elements_(num_elements), + num_categories_(num_categories), + replacement_(replacement) {} + +Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); }//在ShardCategory空间中创建Status型Execute函数,返回值为Status空间中的OK函数的返回值 +//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为0或-1 +//判断dataset_size的值 +//若dataset_size的值为0,则返回dataset_size本身 +//若dataset_size的值大于0,则继续判断num_classes、num_categories_、num_elements_的值是否大于0。若均大于0,则修改num_classes的赋值,赋值为num_categories_和num_classes的最小值 +//继续判断num_classes是否为0,若为0则返回0。不为0则判断num_elements_是否大于int64_t类型最大值与num_classes的商,若是则返回-1。若均不符合则返回num_classes和num_elements_的乘积 +//若均不符合,则返回0 +int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (dataset_size == 0) return dataset_size; + if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { + num_classes = std::min(num_categories_, num_classes); + if (num_classes == 0) { + return 0; + } + if (num_elements_ > std::numeric_limits::max() / num_classes) { + return -1; + } + return num_classes * num_elements_; + } + return 0; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 8b2440376c4981e417df4e715862bd6f24952086 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:02:19 +0800 Subject: [PATCH 123/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_column.cc | 516 ++++++++++++++++++ 1 file changed, 516 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_column.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_column.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_column.cc new file mode 100644 index 00000000000..08931af66c7 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_column.cc @@ -0,0 +1,516 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_column.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "utils/ms_utils.h" +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_error.h" + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardColumn空间中ShardColumn函数,输入两种参数调用指定函数 +//建立schema列表的表头 +ShardColumn::ShardColumn(const std::shared_ptr &shard_header, bool compress_integer) { + auto first_schema = shard_header->GetSchemas()[0]; + json schema_json = first_schema->GetSchema(); + Init(schema_json, compress_integer); +} +//引用ShardColumn空间中Execute函数 +//引用Init函数进行操作 +ShardColumn::ShardColumn(const json &schema_json, bool compress_integer) { Init(schema_json, compress_integer); } +//引用ShardColumn空间中Init函数 +void ShardColumn::Init(const json &schema_json, bool compress_integer) { + auto schema = schema_json["schema"]; + auto blob_fields = schema_json["blob_fields"]; + //进入循环，遍历schema列表，记录key变量 + bool has_integer_array = false; + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + const std::string &column_name = it.key(); + column_name_.push_back(column_name); + + json it_value = it.value(); + //判断shape是否为it_value列表的最末端，若不是则复制it_value列表加入column_shape_中，继续判断str_type是否是int32或int64类型，若是则记has_integer_array为真 + //若不是则将vec转换为int64并记入column_shape_列表中 + std::string str_type = it_value["type"]; + column_data_type_.push_back(ColumnDataTypeMap.at(str_type)); + if (it_value.find("shape") != it_value.end()) { + std::vector vec(it_value["shape"].size()); + std::copy(it_value["shape"].begin(), it_value["shape"].end(), vec.begin()); + column_shape_.push_back(vec); + if (str_type == "int32" || str_type == "int64") { + has_integer_array = true; + } + } else { + std::vector vec = {}; + column_shape_.push_back(vec); + } + } + //创建column_name_id_列表 + for (uint64_t i = 0; i < column_name_.size(); i++) { + column_name_id_[column_name_[i]] = i; + } + //创建blob_column_列表 + for (const auto &field : blob_fields) { + blob_column_.push_back(field); + } + //创建blob_column_id_列表 + for (uint64_t i = 0; i < blob_column_.size(); i++) { + blob_column_id_[blob_column_[i]] = i; + } + + has_compress_blob_ = (compress_integer && has_integer_array); + num_blob_column_ = blob_column_.size(); +} +//引用ShardColumn空间中GetColumnTypeByName函数 +Status ShardColumn::GetColumnTypeByName(const std::string &column_name, ColumnDataType *column_data_type, + uint64_t *column_data_type_size, std::vector *column_shape, + ColumnCategory *column_category) { + RETURN_UNEXPECTED_IF_NULL(column_data_type); + RETURN_UNEXPECTED_IF_NULL(column_data_type_size); + RETURN_UNEXPECTED_IF_NULL(column_shape); + RETURN_UNEXPECTED_IF_NULL(column_category); + // Skip if column not found如果找不到列，则跳过 + *column_category = CheckColumnName(column_name); + CHECK_FAIL_RETURN_UNEXPECTED(*column_category != ColumnNotFound, + "[Internal ERROR] the type of column: " + column_name + " can not found."); + + // Get data type and size获取数据类型和大小 + auto column_id = column_name_id_[column_name]; + *column_data_type = column_data_type_[column_id]; + *column_data_type_size = ColumnDataTypeSize[*column_data_type]; + *column_shape = column_shape_[column_id]; + return Status::OK(); +} +//引用ShardColumn空间中GetColumnValueByName函数 +Status ShardColumn::GetColumnValueByName(const std::string &column_name, const std::vector &columns_blob, + const json &columns_json, const unsigned char **data, + std::unique_ptr *data_ptr, uint64_t *const n_bytes, + ColumnDataType *column_data_type, uint64_t *column_data_type_size, + std::vector *column_shape) { + RETURN_UNEXPECTED_IF_NULL(column_data_type); + RETURN_UNEXPECTED_IF_NULL(column_data_type_size); + RETURN_UNEXPECTED_IF_NULL(column_shape); + // Skip if column not found如果找不到列，则跳过 + auto column_category = CheckColumnName(column_name); + CHECK_FAIL_RETURN_UNEXPECTED(column_category != ColumnNotFound, + "[Internal ERROR] the type of column: " + column_name + " can not found."); + // Get data type and size获取数据类型和大小 + auto column_id = column_name_id_[column_name]; + *column_data_type = column_data_type_[column_id]; + *column_data_type_size = ColumnDataTypeSize[*column_data_type]; + *column_shape = column_shape_[column_id]; + + // Retrieve value from json从json检索值 + if (column_category == ColumnInRaw) { + RETURN_IF_NOT_OK(GetColumnFromJson(column_name, columns_json, data_ptr, n_bytes)); + *data = reinterpret_cast(data_ptr->get()); + return Status::OK(); + } + + // Retrieve value from blob从blob检索值 + RETURN_IF_NOT_OK(GetColumnFromBlob(column_name, columns_blob, data, data_ptr, n_bytes)); + if (*data == nullptr) { + *data = reinterpret_cast(data_ptr->get()); + } + return Status::OK(); +} +//引用ShardColumn空间中GetColumnFromJson函数 +Status ShardColumn::GetColumnFromJson(const std::string &column_name, const json &columns_json, + std::unique_ptr *data_ptr, uint64_t *n_bytes) { + RETURN_UNEXPECTED_IF_NULL(n_bytes); + RETURN_UNEXPECTED_IF_NULL(data_ptr); + auto column_id = column_name_id_[column_name]; + auto column_data_type = column_data_type_[column_id]; + + // Initialize num bytes初始化（以字节为单位） + *n_bytes = ColumnDataTypeSize[column_data_type]; + auto json_column_value = columns_json[column_name]; + CHECK_FAIL_RETURN_UNEXPECTED(json_column_value.is_string() || json_column_value.is_number(), + "[Internal ERROR] the value of column: " + column_name + + " should be string or number but got: " + json_column_value.dump()); + //通过column_data_type选择操作方法 + switch (column_data_type) { + case ColumnFloat32: { + return GetFloat(data_ptr, json_column_value, false); + } + case ColumnFloat64: { + return GetFloat(data_ptr, json_column_value, true); + } + case ColumnInt32: { + return GetInt(data_ptr, json_column_value); + } + case ColumnInt64: { + return GetInt(data_ptr, json_column_value); + } + default: { + // Convert string to c_str将字符串转换为c_str + std::string tmp_string; + if (json_column_value.is_string()) { + tmp_string = json_column_value.get(); + } else { + tmp_string = json_column_value.dump(); + } + *n_bytes = tmp_string.size(); + auto data = reinterpret_cast(common::SafeCStr(tmp_string)); + *data_ptr = std::make_unique(*n_bytes); + for (uint32_t i = 0; i < *n_bytes; i++) { + (*data_ptr)[i] = *(data + i); + } + break; + } + } + return Status::OK(); +} +//创建函数模板，引用ShardColumn空间中GetFloat函数 +template +Status ShardColumn::GetFloat(std::unique_ptr *data_ptr, const json &json_column_value, + bool use_double) { + RETURN_UNEXPECTED_IF_NULL(data_ptr); + std::unique_ptr array_data = std::make_unique(1); + if (json_column_value.is_number()) { + array_data[0] = json_column_value; + } else { + // Convert string to float将字符串转换为浮点 + try { + if (use_double) { + array_data[0] = json_column_value.get(); + } else { + array_data[0] = json_column_value.get(); + } + } catch (json::exception &e) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + json_column_value.dump() + + " to type float, " + std::string(e.what())); + } + } + //创建data_ptr列表 + auto data = reinterpret_cast(array_data.get()); + *data_ptr = std::make_unique(sizeof(T)); + for (uint32_t i = 0; i < sizeof(T); i++) { + (*data_ptr)[i] = *(data + i); + } + return Status::OK(); +} +//创建函数模板，引用ShardColumn空间中GetInt函数 +template +Status ShardColumn::GetInt(std::unique_ptr *data_ptr, const json &json_column_value) { + RETURN_UNEXPECTED_IF_NULL(data_ptr); + std::unique_ptr array_data = std::make_unique(1); + int64_t temp_value; + bool less_than_zero = false; + //判断json_column_value是否为integer类型，若是则令json_zero为0、temp_value等于json_column_value,并判断json_column_value是否小于0，若是则令less_than_zero为true + //判断json_column_value是否为string类型，若是则令string_value为json_column_value + if (json_column_value.is_number_integer()) { + const json json_zero = 0; + if (json_column_value < json_zero) { + less_than_zero = true; + } + temp_value = json_column_value; + } else if (json_column_value.is_string()) { + std::string string_value = json_column_value; + //设置异常捕捉器 + try { + //判断string_value是否为空且string_value列表的第一位是否为‘-’,若是则给temp_value和less_than_zero赋值 + //若不是则直接给temp_value赋值 + if (!string_value.empty() && string_value[0] == '-') { + temp_value = std::stoll(string_value); + less_than_zero = true; + } else { + temp_value = static_cast(std::stoull(string_value)); + } + } catch (std::invalid_argument &e) {//若问题类型为参数无效，则返回相应错误信息 + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " + + std::string(e.what())); + } catch (std::out_of_range &e) {//若问题类型为超出范围，则返回相应错误信息 + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to convert column value:" + string_value + " to type int, " + + std::string(e.what())); + } + } else {//若均不符合，则返回相应错误信息 + RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value should be type string or number but got: " + + json_column_value.dump()); + } + //进行判断，若符合则返回相应错误信息 + if ((less_than_zero && temp_value < static_cast(std::numeric_limits::min())) || + (!less_than_zero && static_cast(temp_value) > static_cast(std::numeric_limits::max()))) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] column value: " + std::to_string(temp_value) + " is out of range."); + } + array_data[0] = static_cast(temp_value); + //进入循环，建立data_ptr列表 + auto data = reinterpret_cast(array_data.get()); + *data_ptr = std::make_unique(sizeof(T)); + for (uint32_t i = 0; i < sizeof(T); i++) { + (*data_ptr)[i] = *(data + i); + } + return Status::OK(); +} +//引用ShardColumn空间中GetColumnFromBlob函数 +//判断已建立的column_name_id_列表和column_data_type_列表是否相同并进行操作 +Status ShardColumn::GetColumnFromBlob(const std::string &column_name, const std::vector &columns_blob, + const unsigned char **data, std::unique_ptr *data_ptr, + uint64_t *const n_bytes) { + RETURN_UNEXPECTED_IF_NULL(data); + uint64_t offset_address = 0; + auto column_id = column_name_id_[column_name]; + RETURN_IF_NOT_OK(GetColumnAddressInBlock(column_id, columns_blob, n_bytes, &offset_address)); + auto column_data_type = column_data_type_[column_id]; + if (has_compress_blob_ && column_data_type == ColumnInt32) { + RETURN_IF_NOT_OK(UncompressInt(column_id, data_ptr, columns_blob, n_bytes, offset_address)); + } else if (has_compress_blob_ && column_data_type == ColumnInt64) { + RETURN_IF_NOT_OK(UncompressInt(column_id, data_ptr, columns_blob, n_bytes, offset_address)); + } else { + *data = reinterpret_cast(&(columns_blob[offset_address])); + } + + return Status::OK(); +} +//引用ShardColumn空间中GetColumnName函数 +//判断it_column是佛偶为column_name_id_列表的结尾，若是则返回ColumnNotFound +//若不是则返回ColumnInRaw或ColumnInBlob +ColumnCategory ShardColumn::CheckColumnName(const std::string &column_name) { + auto it_column = column_name_id_.find(column_name); + if (it_column == column_name_id_.end()) { + return ColumnNotFound; + } + auto it_blob = blob_column_id_.find(column_name); + return it_blob == blob_column_id_.end() ? ColumnInRaw : ColumnInBlob; +} +//引用ShardColumn空间中CompressBlob函数 +std::vector ShardColumn::CompressBlob(const std::vector &blob, int64_t *compression_size) { + // Skip if no compress columns如果没有压缩列，则跳过 + *compression_size = 0; + if (!CheckCompressBlob()) { + return blob; + } + + std::vector dst_blob; + uint64_t i_src = 0; + for (int64_t i = 0; i < num_blob_column_; i++) { + // Get column data type获取列数据类型 + auto src_data_type = column_data_type_[column_name_id_[blob_column_[i]]]; + auto int_type = src_data_type == ColumnInt32 ? kInt32Type : kInt64Type; + + // Compress and return is blob has 1 column only压缩并返回blob只有1列 + if (num_blob_column_ == 1) { + dst_blob = CompressInt(blob, int_type); + *compression_size = static_cast(blob.size()) - static_cast(dst_blob.size()); + return dst_blob; + } + + // Just copy and continue if column dat type is not int32/int64如果列数据类型不是int32/int64，只需复制并继续 + uint64_t num_bytes = BytesBigToUInt64(blob, i_src, kInt64Type); + if (src_data_type != ColumnInt32 && src_data_type != ColumnInt64) { + dst_blob.insert(dst_blob.end(), blob.begin() + i_src, blob.begin() + i_src + kInt64Len + num_bytes); + i_src += kInt64Len + num_bytes; + continue; + } + + // Get column slice in source blob获取源blob中的列切片 + std::vector blob_slice(blob.begin() + i_src + kInt64Len, blob.begin() + i_src + kInt64Len + num_bytes); + // Compress column压缩列 + auto dst_blob_slice = CompressInt(blob_slice, int_type); + // Get new column size获取新列大小 + auto new_blob_size = UIntToBytesBig(dst_blob_slice.size(), kInt64Type); + // Append new column size附加新列大小 + dst_blob.insert(dst_blob.end(), new_blob_size.begin(), new_blob_size.end()); + // Append new column data附加新列数据 + dst_blob.insert(dst_blob.end(), dst_blob_slice.begin(), dst_blob_slice.end()); + i_src += kInt64Len + num_bytes; + } + MS_LOG(DEBUG) << "Compress blob data from " << blob.size() << " to " << dst_blob.size() << "."; + *compression_size = static_cast(blob.size()) - static_cast(dst_blob.size()); + return dst_blob; +} +//引用ShardColumn空间中CompressBlob函数 +vector ShardColumn::CompressInt(const vector &src_bytes, const IntegerType &int_type) { + uint64_t i_size = kUnsignedOne << static_cast(int_type); + // Get number of elements获取元素数 + uint64_t src_n_int = src_bytes.size() / i_size; + // Calculate bitmap size (bytes)计算位图大小（字节） + uint64_t bitmap_size = (src_n_int + kNumDataOfByte - 1) / kNumDataOfByte; + + // Initialize destination blob, more space than needed, will be resized初始化目标blob，超出所需空间，将调整大小 + vector dst_bytes(kBytesOfColumnLen + bitmap_size + src_bytes.size(), 0); + + // Write number of elements to destination blob将元素数写入目标blob + vector size_by_bytes = UIntToBytesBig(src_n_int, kInt32Type); + for (uint64_t n = 0; n < kBytesOfColumnLen; n++) { + dst_bytes[n] = size_by_bytes[n]; + } + + // Write compressed int写入压缩int + uint64_t i_dst = kBytesOfColumnLen + bitmap_size; + for (uint64_t i = 0; i < src_n_int; i++) { + // Initialize destination data type初始化目标数据类型 + IntegerType dst_int_type = kInt8Type; + // Shift to next int position移到下一个int位置 + uint64_t pos = i * (kUnsignedOne << static_cast(int_type)); + // Narrow down this int缩小这个整数 + int64_t i_n = BytesLittleToMinIntType(src_bytes, pos, int_type, &dst_int_type); + + // Write this int to destination blob将此int写入目标blob + uint64_t u_n = *reinterpret_cast(&i_n); + auto temp_bytes = UIntToBytesLittle(u_n, dst_int_type); + for (uint64_t j = 0; j < (kUnsignedOne << static_cast(dst_int_type)); j++) { + dst_bytes[i_dst++] = temp_bytes[j]; + } + + // Update date type in bit map更新位图中的日期类型 + dst_bytes[i / kNumDataOfByte + kBytesOfColumnLen] |= + (static_cast(dst_int_type) << (kDataTypeBits * (kNumDataOfByte - kUnsignedOne - (i % kNumDataOfByte)))); + } + // Resize destination blob调整目标blob的大小 + dst_bytes.resize(i_dst); + MS_LOG(DEBUG) << "Compress blob field from " << src_bytes.size() << " to " << dst_bytes.size() << "."; + return dst_bytes; +} +//引用ShardColumn空间中GetColumnAddressInBlock函数 +Status ShardColumn::GetColumnAddressInBlock(const uint64_t &column_id, const std::vector &columns_blob, + uint64_t *num_bytes, uint64_t *shift_idx) { + RETURN_UNEXPECTED_IF_NULL(num_bytes); + RETURN_UNEXPECTED_IF_NULL(shift_idx); + //判断num_blob_column_是否为1，若是则用指针记录columns_blob列表的大小并返回 + if (num_blob_column_ == 1) { + *num_bytes = columns_blob.size(); + *shift_idx = 0; + return Status::OK(); + } + auto blob_id = blob_column_id_[column_name_[column_id]]; + //进入循环，按步骤调用BytesBigToUInt64 + for (int32_t i = 0; i < blob_id; i++) { + *shift_idx += kInt64Len + BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type); + } + *num_bytes = BytesBigToUInt64(columns_blob, *shift_idx, kInt64Type); + + (*shift_idx) += kInt64Len; + + return Status::OK(); +} +//创建函数模板，引用ShardColumn空间中UncompressInt函数 +template +Status ShardColumn::UncompressInt(const uint64_t &column_id, std::unique_ptr *const data_ptr, + const std::vector &columns_blob, uint64_t *num_bytes, uint64_t shift_idx) { + RETURN_UNEXPECTED_IF_NULL(data_ptr); + RETURN_UNEXPECTED_IF_NULL(num_bytes); + auto num_elements = BytesBigToUInt64(columns_blob, shift_idx, kInt32Type); + *num_bytes = sizeof(T) * num_elements; + + // Parse integer array解析整数数组 + uint64_t i_source = shift_idx + kBytesOfColumnLen + (num_elements + kNumDataOfByte - 1) / kNumDataOfByte; + auto array_data = std::make_unique(num_elements); + + for (uint64_t i = 0; i < num_elements; i++) { + uint8_t iBitMap = columns_blob[shift_idx + kBytesOfColumnLen + i / kNumDataOfByte]; + uint64_t i_type = (iBitMap >> ((kNumDataOfByte - 1 - (i % kNumDataOfByte)) * kDataTypeBits)) & kDataTypeBitMask; + auto mr_int_type = static_cast(i_type); + int64_t i64 = BytesLittleToMinIntType(columns_blob, i_source, mr_int_type); + i_source += (kUnsignedOne << i_type); + array_data[i] = static_cast(i64); + } + + auto data = reinterpret_cast(array_data.get()); + *data_ptr = std::make_unique(*num_bytes); + // field is none. for example: numpy is null字段为无。例如：numpy为null + if (*num_bytes == 0) { + return Status::OK(); + } + CHECK_FAIL_RETURN_UNEXPECTED(memcpy_s(data_ptr->get(), *num_bytes, data, *num_bytes) == 0, + "[Internal ERROR] Failed to call securec func [memcpy_s]"); + return Status::OK(); +} +//引用ShardColumn空间中BytesBigToUInt64函数 +//进入循环，计算result的值并返回 +uint64_t ShardColumn::BytesBigToUInt64(const std::vector &bytes_array, const uint64_t &pos, + const IntegerType &i_type) { + uint64_t result = 0; + for (uint64_t i = 0; i < (kUnsignedOne << static_cast(i_type)); i++) { + result = (result << kBitsOfByte) + bytes_array[pos + i]; + } + return result; +} +//引用ShardColumn空间中UIntToBytesBig函数 +//进入循环，根据操作计算result的值并返回 +std::vector ShardColumn::UIntToBytesBig(uint64_t value, const IntegerType &i_type) { + uint64_t n_bytes = kUnsignedOne << static_cast(i_type); + std::vector result(n_bytes, 0); + for (uint64_t i = 0; i < n_bytes; i++) { + result[n_bytes - 1 - i] = value & std::numeric_limits::max(); + value >>= kBitsOfByte; + } + return result; +} +//引用ShardColumn空间中UIntToBytesLittle函数 +//进入循环，根据操作计算result的值并返回 +std::vector ShardColumn::UIntToBytesLittle(uint64_t value, const IntegerType &i_type) { + uint64_t n_bytes = kUnsignedOne << static_cast(i_type); + std::vector result(n_bytes, 0); + for (uint64_t i = 0; i < n_bytes; i++) { + result[i] = value & std::numeric_limits::max(); + value >>= kBitsOfByte; + } + return result; +} +//引用ShardColumn空间中BytesLittleToMinIntType函数 +int64_t ShardColumn::BytesLittleToMinIntType(const std::vector &bytes_array, const uint64_t &pos, + const IntegerType &src_i_type, IntegerType *dst_i_type) { + uint64_t u_temp = 0; + //进入循环，计算u_temp的值 + for (uint64_t i = 0; i < (kUnsignedOne << static_cast(src_i_type)); i++) { + u_temp = (u_temp << kBitsOfByte) + + bytes_array[pos + (kUnsignedOne << static_cast(src_i_type)) - kUnsignedOne - i]; + } + //根据src_i_type的类型判断，并选择相应的处理 + int64_t i_out; + switch (src_i_type) { + case kInt8Type: { + i_out = (int8_t)(u_temp & std::numeric_limits::max()); + break; + } + case kInt16Type: { + i_out = (int16_t)(u_temp & std::numeric_limits::max()); + break; + } + case kInt32Type: { + i_out = (int32_t)(u_temp & std::numeric_limits::max()); + break; + } + case kInt64Type: { + i_out = (int64_t)(u_temp & std::numeric_limits::max()); + break; + } + default: { + i_out = 0; + } + } + //判断dst_i_type是否为假，若是则直接返回i_out + if (!dst_i_type) { + return i_out; + } + //判断i_out的取值，给指针dst_i_type赋值 + if (i_out >= static_cast(std::numeric_limits::min()) && + i_out <= static_cast(std::numeric_limits::max())) { + *dst_i_type = kInt8Type; + } else if (i_out >= static_cast(std::numeric_limits::min()) && + i_out <= static_cast(std::numeric_limits::max())) { + *dst_i_type = kInt16Type; + } else if (i_out >= static_cast(std::numeric_limits::min()) && + i_out <= static_cast(std::numeric_limits::max())) { + *dst_i_type = kInt32Type; + } else { + *dst_i_type = kInt64Type; + } + return i_out; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 339caec3c45a60bb89dd4bf0ef9125b76e1fd555 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:02:34 +0800 Subject: [PATCH 124/160] ADD file via upload --- .../meta/shard_distributed_sample.cc | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_distributed_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_distributed_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_distributed_sample.cc new file mode 100644 index 00000000000..eea741770e0 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_distributed_sample.cc @@ -0,0 +1,95 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_distributed_sample.h"//按照路径寻找以下文件，导入到本文件 + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardDistributedSample空间中ShardDistributedSample函数,输入七种参数调用指定函数 +//对参数ShardSample、shuffle_、no_of_padded_samples_、first_epoch_、shuffle_op_进行进行参数初始化 +ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, int64_t no_of_padded_samples, bool shuffle, + uint32_t seed, int64_t no_of_samples, int64_t offset) + : ShardSample(1, num_shards, shard_id, no_of_samples, offset), + shuffle_(shuffle), + no_of_padded_samples_(no_of_padded_samples), + first_epoch_(true) { + shuffle_op_ = std::make_shared(seed, kShuffleSample); +} +//引用ShardDistributedSample空间中ShardDistributedSample函数,输入六种参数调用指定函数 +//对参数ShardDistributedSample进行进行参数初始化 +ShardDistributedSample::ShardDistributedSample(int num_shards, int shard_id, bool shuffle, uint32_t seed, + int64_t no_of_samples, int64_t offset) + : ShardDistributedSample(num_shards, shard_id, 0, shuffle, seed, no_of_samples, offset) {} +//在ShardDistributedSample空间中创建int64_t型GetNumSamples函数,返回值为0或-1 +//判断no_of_padded_samples_的值 +//若no_of_padded_samples_的值小于等于0,令res变量等于0。若no_of_padded_samples_大于0,则返回0。 +//判断dataset_size与denominator_的模是否等于0,若相等则将dataset_size与denominator_和numerator_的乘积的商赋给res +//若不相等,则将dataset_size与denominator_和numerator_的乘积的商+1后赋给res +//返回no_of_samples_,no_of_samples_的取值取决于res是否等于0,若等于0则返回0,若不等于0则返回no_of_samples_与res的最小值 +//若no_of_padded_samples_的值大于0,将dataset_size和no_of_padded_samples_的和赋给padded_size +//判断padded_size与denominator_的模是否为0,若是则返回padded_size与denominator_和numerator_的乘积的商,若不是则返回-1 +int64_t ShardDistributedSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (no_of_padded_samples_ <= 0) { + int64_t res = 0; + if (dataset_size % denominator_ == 0) { + res = dataset_size / denominator_ * numerator_; + } else { + res = dataset_size / denominator_ * numerator_ + 1; + } + return no_of_samples_ == 0 ? res : std::min(no_of_samples_, res); + } else { + auto padded_size = dataset_size + no_of_padded_samples_; + if (padded_size % denominator_ == 0) { + return padded_size / denominator_ * numerator_; + } else { + return -1; + } + } + return 0; +} +//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量 +//将tasks.Size()函数的返回值赋给total_no +//判断no_of_padded_samples_是否大于0且first_epoch_是否为真,若是则调用CHECK_FAIL_RETURN_UNEXPECTED函数并输出非有效数据警告 +//若no_of_padded_samples_小于0,则判断first_epoch_是否为真,若是则将first_epoch_改为否,将tasks赋给task_。若不是则将task_赋给tasks +//判断shuffle_是否为真，若是则调用shuffle_op_指针中SetShardSampleCount和UpdateShuffleMode的函数并调用RETURN_IF_NOT_OK函数 +//返回Status中Ok函数的返回值 +Status ShardDistributedSample::PreExecute(ShardTaskList &tasks) { + auto total_no = tasks.Size(); + if (no_of_padded_samples_ > 0 && first_epoch_) { + CHECK_FAIL_RETURN_UNEXPECTED(total_no % denominator_ == 0, + "Invalid data, the size of dataset and padded samples: " + std::to_string(total_no) + + " can not be divisible by the value of 'num_shards': " + + std::to_string(denominator_) + ".\n Please adjust the value of 'num_padded'."); + } + if (first_epoch_) { + first_epoch_ = false; + task_ = tasks; + } else { + tasks = task_; + } + if (shuffle_ == true) { + shuffle_op_->SetShardSampleCount(GetShardSampleCount()); + shuffle_op_->UpdateShuffleMode(GetShuffleMode()); + RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 3b4c0d98a00941de07454e022ead259262c12965 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:02:51 +0800 Subject: [PATCH 125/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_header.cc | 724 ++++++++++++++++++ 1 file changed, 724 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_header.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_header.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_header.cc new file mode 100644 index 00000000000..54dd388db00 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_header.cc @@ -0,0 +1,724 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_header.h"//按照路径寻找以下文件，导入到本文件 + +#include // +#include +#include +#include +#include + +#include "utils/file_utils.h"//按照路径寻找以下文件，导入到本文件 +#include "utils/ms_utils.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "minddata/mindrecord/include/shard_page.h" + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//在atomic空间中创建bool型thread_status函数 +std::atomic thread_status(false); +//引用ShardHeader空间中ShardHeader函数,输入三种参数调用指定函数 +//对参数shard_count_、header_size_、page_size_、compression_size_进行进行参数初始化 +//给index_变量赋值，值为访问Index空间的数量 +ShardHeader::ShardHeader() : shard_count_(0), header_size_(0), page_size_(0), compression_size_(0) { + index_ = std::make_shared(); +} +//在ShardHeader空间中创建InitializeHeader函数 +Status ShardHeader::InitializeHeader(const std::vector &headers, bool load_dataset) { + shard_count_ = headers.size(); + int shard_index = 0; + bool first = true; + //进入循环，判断first是否为真，若是则进行header列表的创建 + for (const auto &header : headers) { + if (first) { + first = false; + RETURN_IF_NOT_OK(ParseSchema(header["schema"])); + RETURN_IF_NOT_OK(ParseIndexFields(header["index_fields"])); + RETURN_IF_NOT_OK(ParseStatistics(header["statistics"])); + ParseShardAddress(header["shard_addresses"]); + header_size_ = header["header_size"].get(); + page_size_ = header["page_size"].get(); + compression_size_ = header.contains("compression_size") ? header["compression_size"].get() : 0; + } + RETURN_IF_NOT_OK(ParsePage(header["page"], shard_index, load_dataset)); + shard_index++; + } + return Status::OK(); +} +//在ShardHeader空间中创建CheckFileStatus函数 +Status ShardHeader::CheckFileStatus(const std::string &path) { + auto realpath = FileUtils::GetRealPath(path.c_str()); + //捕捉错误，并输出错误信息 + CHECK_FAIL_RETURN_UNEXPECTED( + realpath.has_value(), + "Invalid file, failed to get the realpath of mindrecord files. Please check file path: " + path); + std::ifstream fin(realpath.value(), std::ios::in | std::ios::binary); + CHECK_FAIL_RETURN_UNEXPECTED(fin.is_open(), + "Invalid file, failed to open files for loading mindrecord files. Please check file " + "path, permission and open file limit: " + + path); + // fetch file size获取文件大小 + auto &io_seekg = fin.seekg(0, std::ios::end); + if (!io_seekg.good() || io_seekg.fail() || io_seekg.bad()) { + fin.close(); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] failed to seekg file, file path: " + path); + } + //捕捉错误，并输出错误信息 + size_t file_size = fin.tellg(); + if (file_size < kMinFileSize) { + fin.close(); + RETURN_STATUS_UNEXPECTED("Invalid file, the size of mindrecord file: " + std::to_string(file_size) + + " is smaller than the lower limit: " + std::to_string(kMinFileSize) + + ".\n Please check file path: " + path + + " and use 'FileWriter' to generate valid mindrecord files."); + } + fin.close(); + return Status::OK(); +} +//在ShardHeader空间中创建ValidateHeader函数 +Status ShardHeader::ValidateHeader(const std::string &path, std::shared_ptr *header_ptr) { + RETURN_UNEXPECTED_IF_NULL(header_ptr); + RETURN_IF_NOT_OK(CheckFileStatus(path)); + // + auto realpath = FileUtils::GetRealPath(path.c_str()); + CHECK_FAIL_RETURN_UNEXPECTED( + realpath.has_value(), + "Invalid file, failed to get the realpath of mindrecord files. Please check file path: " + path); + + // read header size读取标头大小 + json json_header; + std::ifstream fin(realpath.value(), std::ios::in | std::ios::binary); + CHECK_FAIL_RETURN_UNEXPECTED(fin.is_open(), + "Invalid file, failed to open files for loading mindrecord files. Please check file " + "path, permission and open file limit: " + + path); + //判断io_rea的相关参数，若符合则关闭fin，并返回错误信息 + uint64_t header_size = 0; + auto &io_read = fin.read(reinterpret_cast(&header_size), kInt64Len); + if (!io_read.good() || io_read.fail() || io_read.bad()) { + fin.close(); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] failed to read file, file path: " + path); + } + //判断header列表的大小是否大于header应有的大小 + //若符合则关闭fin并返回错误信息 + if (header_size > kMaxHeaderSize) { + fin.close(); + RETURN_STATUS_UNEXPECTED( + "Invalid file, the size of mindrecord file header is larger than the upper limit. \nPlease use 'FileWriter' to " + "generate valid mindrecord files."); + } + + // read header content读取标题内容 + std::vector header_content(header_size); + auto &io_read_content = fin.read(reinterpret_cast(&header_content[0]), header_size); + if (!io_read_content.good() || io_read_content.fail() || io_read_content.bad()) { + fin.close(); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to read file, file path: " + path); + } + + fin.close(); + std::string raw_header_content = std::string(header_content.begin(), header_content.end()); + // parse json content解析json内容 + try { + json_header = json::parse(raw_header_content); + } catch (json::parse_error &e) { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to parse the metadata in JSON format in the mindrecord files: " + + std::string(e.what())); + } + *header_ptr = std::make_shared(json_header); + return Status::OK(); +} +//引用ShardHeader空间中BuildSingleHeader函数 +//创建singleheader列表 +Status ShardHeader::BuildSingleHeader(const std::string &file_path, std::shared_ptr *header_ptr) { + RETURN_UNEXPECTED_IF_NULL(header_ptr); + std::shared_ptr raw_header; + RETURN_IF_NOT_OK(ValidateHeader(file_path, &raw_header)); + uint64_t compression_size = + raw_header->contains("compression_size") ? (*raw_header)["compression_size"].get() : 0; + json header = {{"shard_addresses", (*raw_header)["shard_addresses"]}, + {"header_size", (*raw_header)["header_size"]}, + {"page_size", (*raw_header)["page_size"]}, + {"compression_size", compression_size}, + {"index_fields", (*raw_header)["index_fields"]}, + {"blob_fields", (*raw_header)["schema"][0]["blob_fields"]}, + {"schema", (*raw_header)["schema"][0]["schema"]}, + {"version", (*raw_header)["version"]}}; + *header_ptr = std::make_shared(header); + return Status::OK(); +} +//引用ShardHeader空间中BuildDataset函数 +//创建dataset列表 +Status ShardHeader::BuildDataset(const std::vector &file_paths, bool load_dataset) { + uint32_t thread_num = std::thread::hardware_concurrency(); + if (thread_num == 0) { + thread_num = kThreadNumber; + } + uint32_t work_thread_num = 0; + uint32_t shard_count = file_paths.size(); + int group_num = ceil(shard_count * 1.0 / thread_num); + std::vector thread_set(thread_num); + std::vector headers(shard_count); + for (uint32_t x = 0; x < thread_num; ++x) { + int start_num = x * group_num; + int end_num = ((x + 1) * group_num > shard_count) ? shard_count : (x + 1) * group_num; + if (start_num >= end_num) { + continue; + } + + thread_set[x] = + std::thread(&ShardHeader::GetHeadersOneTask, this, start_num, end_num, std::ref(headers), file_paths); + work_thread_num++; + } + + for (uint32_t x = 0; x < work_thread_num; ++x) { + thread_set[x].join(); + } + if (thread_status) { + thread_status = false; + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Error raised in GetHeadersOneTask function."); + } + RETURN_IF_NOT_OK(InitializeHeader(headers, load_dataset)); + return Status::OK(); +} +//引用ShardHeader空间中GetHeadersOneTask函数 +void ShardHeader::GetHeadersOneTask(int start, int end, std::vector &headers, + const vector &realAddresses) { + if (thread_status || end > realAddresses.size()) { + return; + } + for (int x = start; x < end; ++x) { + std::shared_ptr header; + auto status = ValidateHeader(realAddresses[x], &header); + if (status.IsError()) { + thread_status = true; + return; + } + (*header)["shard_addresses"] = realAddresses; + if (std::find(kSupportedVersion.begin(), kSupportedVersion.end(), (*header)["version"]) == + kSupportedVersion.end()) { + MS_LOG(ERROR) << "Invalid file, the version of mindrecord files" << (*header)["version"].dump() + << " is not supported.\nPlease use 'FileWriter' to generate valid mindrecord files."; + thread_status = true; + return; + } + headers[x] = *header; + } +} +//引用ShardHeader空间中InitByFiles函数 +Status ShardHeader::InitByFiles(const std::vector &file_paths) { + std::vector file_names(file_paths.size()); + std::transform(file_paths.begin(), file_paths.end(), file_names.begin(), [](std::string fp) -> std::string { + std::shared_ptr fn; + return GetFileName(fp, &fn).IsOk() ? *fn : ""; + }); + + shard_addresses_ = std::move(file_names); + shard_count_ = file_paths.size(); + CHECK_FAIL_RETURN_UNEXPECTED(shard_count_ != 0 && (shard_count_ <= kMaxShardCount), + "[Internal ERROR] 'shard_count_': " + std::to_string(shard_count_) + + "is not in range (0, " + std::to_string(kMaxShardCount) + "]."); + pages_.resize(shard_count_); + return Status::OK(); +} +//引用ShardHeader空间中ParseIndexFields函数 +//进入循环遍历index_field列表，获得parsed_index_fields列表 +Status ShardHeader::ParseIndexFields(const json &index_fields) { + std::vector> parsed_index_fields; + for (auto &index_field : index_fields) { + auto schema_id = index_field["schema_id"].get(); + std::string field_name = index_field["index_field"].get(); + std::pair parsed_index_field(schema_id, field_name); + parsed_index_fields.push_back(parsed_index_field); + } + RETURN_IF_NOT_OK(AddIndexFields(parsed_index_fields)); + return Status::OK(); +} +//引用ShardHeader空间中ParsePage函数 +Status ShardHeader::ParsePage(const json &pages, int shard_index, bool load_dataset) { + // set shard_index when load_dataset is false当load_dataset为false时设置shard_index + CHECK_FAIL_RETURN_UNEXPECTED(shard_count_ <= kMaxFileCount, + "Invalid file, the number of mindrecord files: " + std::to_string(shard_count_) + + "is not in range (0, " + std::to_string(kMaxFileCount) + + "].\nPlease use 'FileWriter' to generate fewer mindrecord files."); + //判断pages_列表是否为空，若是则更改pages_列表的大小 + if (pages_.empty()) { + pages_.resize(shard_count_); + } + //进入循环遍历pages_列表，设置pages_列表内容 + for (auto &page : pages) { + int page_id = page["page_id"]; + int shard_id = page["shard_id"]; + std::string page_type = page["page_type"]; + int page_type_id = page["page_type_id"]; + auto start_row_id = page["start_row_id"].get(); + auto end_row_id = page["end_row_id"].get(); + + std::vector> row_group_ids(page["row_group_ids"].size()); + std::transform(page["row_group_ids"].begin(), page["row_group_ids"].end(), row_group_ids.begin(), + [](json rg) { return std::make_pair(rg["id"], rg["offset"].get()); }); + + auto page_size = page["page_size"].get(); + + std::shared_ptr parsed_page = std::make_shared(page_id, shard_id, page_type, page_type_id, start_row_id, + end_row_id, row_group_ids, page_size); + if (load_dataset == true) { + pages_[shard_id].push_back(std::move(parsed_page)); + } else { + pages_[shard_index].push_back(std::move(parsed_page)); + } + } + return Status::OK(); +} +//引用ShardHeader空间中ParseStatistics函数 +Status ShardHeader::ParseStatistics(const json &statistics) { + //进入循环，遍历statistics列表，若有错误输出错误信息 + for (auto &statistic : statistics) { + CHECK_FAIL_RETURN_UNEXPECTED( + statistic.find("desc") != statistic.end() && statistic.find("statistics") != statistic.end(), + "[Internal ERROR] Failed to deserialize statistics: " + statistics.dump()); + std::string statistic_description = statistic["desc"].get(); + json statistic_body = statistic["statistics"]; + std::shared_ptr parsed_statistic = Statistics::Build(statistic_description, statistic_body); + RETURN_UNEXPECTED_IF_NULL(parsed_statistic); + AddStatistic(parsed_statistic); + } + return Status::OK(); +} +//引用ShardHeader空间中ParseShardAddress函数 +Status ShardHeader::ParseSchema(const json &schemas) { + for (auto &schema : schemas) { + // change how we get schemaBody once design is finalized设计完成后，更改获取schemaBody的方式 + CHECK_FAIL_RETURN_UNEXPECTED(schema.find("desc") != schema.end() && schema.find("blob_fields") != schema.end() && + schema.find("schema") != schema.end(), + "[Internal ERROR] Failed to deserialize schema: " + schema.dump()); + std::string schema_description = schema["desc"].get(); + std::vector blob_fields = schema["blob_fields"].get>(); + json schema_body = schema["schema"]; + std::shared_ptr parsed_schema = Schema::Build(schema_description, schema_body); + RETURN_UNEXPECTED_IF_NULL(parsed_schema); + AddSchema(parsed_schema); + } + return Status::OK(); +} +//引用ShardHeader空间中ParseShardAddress函数 +//复制address列表到shard_addresses_ +void ShardHeader::ParseShardAddress(const json &address) { + std::copy(address.begin(), address.end(), std::back_inserter(shard_addresses_)); +} +//引用ShardHeader空间中SerializeHeader函数 +std::vector ShardHeader::SerializeHeader() { + std::vector header; + auto index = SerializeIndexFields(); + auto stats = SerializeStatistics(); + auto schema = SerializeSchema(); + auto pages = SerializePage(); + auto address = SerializeShardAddress(); + //判断shard_count_是否符合条件，若符合则返回string类型 + if (shard_count_ > static_cast(pages.size())) { + return std::vector{}; + } + //判断shard_count_是否小于shard_count_最大值，若是则进入循环，编辑s字符串，最后返回字符串信息 + if (shard_count_ <= kMaxShardCount) { + for (int shardId = 0; shardId < shard_count_; shardId++) { + string s; + s += "{\"header_size\":" + std::to_string(header_size_) + ","; + s += "\"index_fields\":" + index + ","; + s += "\"page\":" + pages[shardId] + ","; + s += "\"page_size\":" + std::to_string(page_size_) + ","; + s += "\"compression_size\":" + std::to_string(compression_size_) + ","; + s += "\"schema\":" + schema + ","; + s += "\"shard_addresses\":" + address + ","; + s += "\"shard_id\":" + std::to_string(shardId) + ","; + s += "\"statistics\":" + stats + ","; + s += "\"version\":\"" + std::string(kVersion) + "\""; + s += "}"; + header.emplace_back(s); + } + } + return header; +} +//引用ShardHeader空间中SerializeIndexFields函数 +//遍历fields列表，返回schema_id、index_field的值 +std::string ShardHeader::SerializeIndexFields() { + json j; + auto fields = index_->GetFields(); + (void)std::transform(fields.begin(), fields.end(), std::back_inserter(j), + [](const std::pair &field) -> json { + return {{"schema_id", field.first}, {"index_field", field.second}}; + }); + return j.dump(); +} +//引用ShardHeader空间中SerializePage函数 +//遍历shard_pages列表，返回GetPage函数的值 +std::vector ShardHeader::SerializePage() { + std::vector pages; + for (auto &shard_pages : pages_) { + json j; + (void)std::transform(shard_pages.begin(), shard_pages.end(), std::back_inserter(j), + [](const std::shared_ptr &p) { return p->GetPage(); }); + pages.emplace_back(j.dump()); + } + return pages; +} +//引用ShardHeader空间中SerializeStatistics函数 +//遍历statistics_列表，返回GetStatistics函数的值 +std::string ShardHeader::SerializeStatistics() { + json j; + (void)std::transform(statistics_.begin(), statistics_.end(), std::back_inserter(j), + [](const std::shared_ptr &stats) { return stats->GetStatistics(); }); + return j.dump(); +} +//引用ShardHeader空间中SerializeSchema函数 +//遍历schema_列表，返回GetSchema函数的值 +std::string ShardHeader::SerializeSchema() { + json j; + (void)std::transform(schema_.begin(), schema_.end(), std::back_inserter(j), + [](const std::shared_ptr &schema) { return schema->GetSchema(); }); + return j.dump(); +} +//引用ShardHeader空间中SerializeShardAddress函数 +//进入循环，一次根据地址获得文件名并判断是否为空 +std::string ShardHeader::SerializeShardAddress() { + json j; + std::shared_ptr fn_ptr; + for (const auto &addr : shard_addresses_) { + (void)GetFileName(addr, &fn_ptr); + (void)j.emplace_back(*fn_ptr); + } + return j.dump(); +} +//引用ShardHeader空间中GetPage函数 +//判断shard_id、page_id是否在标准范围内，若是则获取pages_列表中的对应信息，若不是则返回错误信息 +Status ShardHeader::GetPage(const int &shard_id, const int &page_id, std::shared_ptr *page_ptr) { + RETURN_UNEXPECTED_IF_NULL(page_ptr); + if (shard_id < static_cast(pages_.size()) && page_id < static_cast(pages_[shard_id].size())) { + *page_ptr = pages_[shard_id][page_id]; + return Status::OK(); + } + page_ptr = nullptr; + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to get Page, 'page_id': " + std::to_string(page_id)); +} +//引用ShardHeader空间中SetPage函数 +//判断shard_id、page_id是否在标准范围内，若是则在pages_列表中加入新值，若不是则返回错误信息 +Status ShardHeader::SetPage(const std::shared_ptr &new_page) { + int shard_id = new_page->GetShardID(); + int page_id = new_page->GetPageID(); + if (shard_id < static_cast(pages_.size()) && page_id < static_cast(pages_[shard_id].size())) { + pages_[shard_id][page_id] = new_page; + return Status::OK(); + } + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to set Page, 'page_id': " + std::to_string(page_id)); +} +//引用ShardHeader空间中AddPage函数 +//判断shard_id、page_id是否在标准范围内，若是则在pages_列表中添加page，若不是则返回错误信息 +Status ShardHeader::AddPage(const std::shared_ptr &new_page) { + int shard_id = new_page->GetShardID(); + int page_id = new_page->GetPageID(); + if (shard_id < static_cast(pages_.size()) && page_id == static_cast(pages_[shard_id].size())) { + pages_[shard_id].push_back(new_page); + return Status::OK(); + } + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to add Page, 'page_id': " + std::to_string(page_id)); +} +//引用ShardHeader空间中GetLastPageId函数 +//判断shard_id是否在范围内，若是则返回0，若不是则返回pages_列表的大小-1的值 +int64_t ShardHeader::GetLastPageId(const int &shard_id) { + if (shard_id >= static_cast(pages_.size())) { + return 0; + } + return pages_[shard_id].size() - 1; +} +//引用ShardHeader空间中GetLastPageIdByType函数 +int ShardHeader::GetLastPageIdByType(const int &shard_id, const std::string &page_type) { + //判断shard_id是否在范围内，若是则返回0 + if (shard_id >= static_cast(pages_.size())) { + return 0; + } + int last_page_id = -1; + //进入循环，若pages_列表中存在page_type则给last_page_id 赋值 + for (uint64_t i = pages_[shard_id].size(); i >= 1; i--) { + if (pages_[shard_id][i - 1]->GetPageType() == page_type) { + last_page_id = pages_[shard_id][i - 1]->GetPageID(); + return last_page_id; + } + } + return last_page_id; +} +//引用ShardHeader空间中GetPageByGroupId函数 +//若发现错误则输出错误信息 +//进入循环，建立page_ptr链表 +Status ShardHeader::GetPageByGroupId(const int &group_id, const int &shard_id, std::shared_ptr *page_ptr) { + RETURN_UNEXPECTED_IF_NULL(page_ptr); + CHECK_FAIL_RETURN_UNEXPECTED(shard_id < static_cast(pages_.size()), + "[Internal ERROR] 'shard_id': " + std::to_string(shard_id) + + " should be smaller than the size of 'pages_': " + std::to_string(pages_.size()) + + "."); + for (uint64_t i = pages_[shard_id].size(); i >= 1; i--) { + auto page = pages_[shard_id][i - 1]; + if (page->GetPageType() == kPageTypeBlob && page->GetPageTypeID() == group_id) { + *page_ptr = std::make_shared(*page); + return Status::OK(); + } + } + page_ptr = nullptr; + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to get Page, 'group_id': " + std::to_string(group_id)); +} +//引用ShardHeader空间中AddSchema函数 +int ShardHeader::AddSchema(std::shared_ptr schema) { + //判断schema列表是否为空，若是则返回错误信息 + if (schema == nullptr) { + MS_LOG(ERROR) << "[Internal ERROR] The pointer of schema is NULL."; + return -1; + } + //继续判断schema空间中的存储空间是否为空，若是则返回错误信息 + if (!schema_.empty()) { + MS_LOG(ERROR) << "The schema is added repeatedly. Please remove the redundant 'add_schema' function."; + return -1; + } + //判断schema_id是否为-1，若是则给schema_id再赋值，后加入到schema列表 + int64_t schema_id = schema->GetSchemaID(); + if (schema_id == -1) { + schema_id = schema_.size(); + schema->SetSchemaID(schema_id); + } + schema_.push_back(schema); + return schema_id; +} +//引用ShardHeader空间中AddStatistic函数 +//判断statistic是否为真，若是则对statistic_id赋值，并将值列入statistic列表 +void ShardHeader::AddStatistic(std::shared_ptr statistic) { + if (statistic) { + int64_t statistics_id = statistic->GetStatisticsID(); + if (statistics_id == -1) { + statistics_id = statistics_.size(); + statistic->SetStatisticsID(statistics_id); + } + statistics_.push_back(statistic); + } +} +//引用ShardHeader空间中InitIndexPtr函数 +//判断index_是否为真，若是则进行操作 +std::shared_ptr ShardHeader::InitIndexPtr() { + std::shared_ptr index = index_; + if (!index_) { + index = std::make_shared(); + index_ = index; + } + return index; +} +//引用ShardHeader空间中CheckIndexField函数,输入参数n调用指定函数 +Status ShardHeader::CheckIndexField(const std::string &field, const json &schema) { + // check field name is or is not valid检查字段名称无效 + CHECK_FAIL_RETURN_UNEXPECTED(schema.find(field) != schema.end(), + "Invalid input, 'index_fields': " + field + " can not found in schema: " + + schema.dump() + ".\n Please use 'add_index' function to add proper 'index_fields'."); + CHECK_FAIL_RETURN_UNEXPECTED(schema[field]["type"] != "Bytes", + "Invalid input, type of 'index_fields': " + field + + " is bytes and can not set as an 'index_fields'.\n Please use 'add_index' function to " + "add the other 'index_fields'."); + CHECK_FAIL_RETURN_UNEXPECTED(schema.find(field) == schema.end() || schema[field].find("shape") == schema[field].end(), + "Invalid input, type of 'index_fields': " + field + + " is array and can not set as an 'index_fields'.\n Please use 'add_index' function to " + "add the other 'index_fields'."); + return Status::OK(); +} +//引用ShardHeader空间中AddIndexFields函数,输入参数n调用指定函数 +Status ShardHeader::AddIndexFields(const std::vector &fields) { + //判断fields是否为空 + if (fields.empty()) { + return Status::OK(); + } + CHECK_FAIL_RETURN_UNEXPECTED(!GetSchemas().empty(), + "Invalid data, schema is empty. Please use 'add_schema' function to add schema first."); + // create index Object创建索引对象 + std::shared_ptr index = InitIndexPtr(); + for (const auto &schemaPtr : schema_) { + std::shared_ptr schema_ptr; + RETURN_IF_NOT_OK(GetSchemaByID(schemaPtr->GetSchemaID(), &schema_ptr)); + json schema = schema_ptr->GetSchema().at("schema"); + // checkout and add fields for each schema签出并为每个架构添加字段 + std::set field_set; + for (const auto &item : index->GetFields()) { + field_set.insert(item.second); + } + for (const auto &field : fields) { + CHECK_FAIL_RETURN_UNEXPECTED( + field_set.find(field) == field_set.end(), + "The 'index_fields': " + field + " is added repeatedly. Please remove the redundant 'add_index' function."); + // check field name is or is not valid检查字段名称无效 + RETURN_IF_NOT_OK(CheckIndexField(field, schema)); + field_set.insert(field); + // add field into index将字段添加到索引中 + index.get()->AddIndexField(schemaPtr->GetSchemaID(), field); + } + } + index_ = index; + return Status::OK(); +} +//引用ShardHeader空间中GetAllSchemaID函数,输入参数n调用指定函数 +Status ShardHeader::GetAllSchemaID(std::set &bucket_count) { + // get all schema id获取所有架构id + for (const auto &schema : schema_) { + auto schema_id = schema->GetSchemaID(); + CHECK_FAIL_RETURN_UNEXPECTED(bucket_count.find(schema_id) == bucket_count.end(), + "[Internal ERROR] duplicate schema exist, schema id: " + std::to_string(schema_id)); + bucket_count.insert(schema_id); + } + return Status::OK(); +} +//引用ShardHeader空间中AddIndexFields函数,输入参数n调用指定函数 +Status ShardHeader::AddIndexFields(std::vector> fields) { + //判断fields是否为空 + if (fields.empty()) { + return Status::OK(); + } + // create index Object创建索引对象 + std::shared_ptr index = InitIndexPtr(); + // get all schema id获取所有架构id + std::set bucket_count; + RETURN_IF_NOT_OK(GetAllSchemaID(bucket_count)); + // check and add fields for each schema检查并添加每个模式的字段 + std::set> field_set; + for (const auto &item : index->GetFields()) { + field_set.insert(item); + } + for (const auto &field : fields) { + CHECK_FAIL_RETURN_UNEXPECTED(field_set.find(field) == field_set.end(), + "The 'index_fields': " + field.second + + " is added repeatedly. Please remove the redundant 'add_index' function."); + uint64_t schema_id = field.first; + std::string field_name = field.second; + + // check schemaId is or is not valid check schemaId无效 + CHECK_FAIL_RETURN_UNEXPECTED(bucket_count.find(schema_id) != bucket_count.end(), + "[Internal ERROR] 'schema_id': " + std::to_string(schema_id) + " can not found."); + // check field name is or is not valid check field name无效 + std::shared_ptr schema_ptr; + RETURN_IF_NOT_OK(GetSchemaByID(schema_id, &schema_ptr)); + json schema = schema_ptr->GetSchema().at("schema"); + CHECK_FAIL_RETURN_UNEXPECTED(schema.find(field_name) != schema.end(), + "Invalid input, 'index_fields': " + field_name + " can not found in schema: " + + schema.dump() + ".\n Please use 'add_index' function to add proper 'index_fields'."); + RETURN_IF_NOT_OK(CheckIndexField(field_name, schema)); + field_set.insert(field); + // add field into index将字段添加到索引中 + index->AddIndexField(schema_id, field_name); + } + index_ = index; + return Status::OK(); +} +//引用ShardHeader空间中GetShardAddressByID函数,输入参数n调用指定函数 +//判断shard_id是否大于shard_addresses_列表的大小，若是则返回“”，若不是则返回shard_addresses_列表中at函数的返回值 +std::string ShardHeader::GetShardAddressByID(int64_t shard_id) { + if (shard_id >= shard_addresses_.size()) { + return ""; + } + return shard_addresses_.at(shard_id); +} +//引用ShardHeader空间中GetSchemas函数,返回schema_的值 +std::vector> ShardHeader::GetSchemas() { return schema_; } +//引用ShardHeader空间中GetStatistics函数,返回statistics_的值 +std::vector> ShardHeader::GetStatistics() { return statistics_; } +//引用ShardHeader空间中GetFields函数,返回index_中GetFields的函数返回值 +std::vector> ShardHeader::GetFields() { return index_->GetFields(); } +//引用ShardHeader空间中GetIndex函数,返回index_的值 +std::shared_ptr ShardHeader::GetIndex() { return index_; } +//引用ShardHeader空间中GetSchemaByID函数 +//判断schema列表中id是否在标准范围内 +Status ShardHeader::GetSchemaByID(int64_t schema_id, std::shared_ptr *schema_ptr) { + RETURN_UNEXPECTED_IF_NULL(schema_ptr); + int64_t schema_size = schema_.size(); + CHECK_FAIL_RETURN_UNEXPECTED(schema_id >= 0 && schema_id < schema_size, + "[Internal ERROR] 'schema_id': " + std::to_string(schema_id) + " is not in range [0, " + + std::to_string(schema_size) + ")."); + *schema_ptr = schema_.at(schema_id); + return Status::OK(); +} +//引用ShardHeader空间中GetStatisticByID函数 +//判断statistic列表中id的值是否在标准范围内 +Status ShardHeader::GetStatisticByID(int64_t statistic_id, std::shared_ptr *statistics_ptr) { + RETURN_UNEXPECTED_IF_NULL(statistics_ptr); + int64_t statistics_size = statistics_.size(); + CHECK_FAIL_RETURN_UNEXPECTED(statistic_id >= 0 && statistic_id < statistics_size, + "[Internal ERROR] 'statistic_id': " + std::to_string(statistic_id) + + " is not in range [0, " + std::to_string(statistics_size) + ")."); + *statistics_ptr = statistics_.at(statistic_id); + return Status::OK(); +} +//在ShardHeader空间中创建PagesToFile函数 +Status ShardHeader::PagesToFile(const std::string dump_file_name) { + auto realpath = FileUtils::GetRealPath(dump_file_name.c_str()); + CHECK_FAIL_RETURN_UNEXPECTED(realpath.has_value(), + "[Internal ERROR] Failed to get the realpath of Pages file, path: " + dump_file_name); + // write header content to file, dump whatever is in the file before将头内容写入文件，转储之前文件中的任何内容 + std::ofstream page_out_handle(realpath.value(), std::ios_base::trunc | std::ios_base::out); + CHECK_FAIL_RETURN_UNEXPECTED(page_out_handle.good(), + "[Internal ERROR] Failed to open Pages file, path: " + dump_file_name); + auto pages = SerializePage(); + for (const auto &shard_pages : pages) { + page_out_handle << shard_pages << "\n"; + } + page_out_handle.close(); + return Status::OK(); +} +//在ShardHeader空间中创建FileToPages函数 +Status ShardHeader::FileToPages(const std::string dump_file_name) { + for (auto &v : pages_) { // clean pages清理页面 + v.clear(); + } + auto realpath = FileUtils::GetRealPath(dump_file_name.c_str()); + CHECK_FAIL_RETURN_UNEXPECTED(realpath.has_value(), + "[Internal ERROR] Failed to get the realpath of Pages file, path: " + dump_file_name); + // attempt to open the file contains the page in json试图打开包含json页面的文件 + std::ifstream page_in_handle(realpath.value()); + CHECK_FAIL_RETURN_UNEXPECTED(page_in_handle.good(), + "[Internal ERROR] Pages file does not exist, path: " + dump_file_name); + std::string line; + while (std::getline(page_in_handle, line)) { + RETURN_IF_NOT_OK(ParsePage(json::parse(line), -1, true)); + } + page_in_handle.close(); + return Status::OK(); +} +//在ShardHeader空间中创建Initialize函数 +Status ShardHeader::Initialize(const std::shared_ptr *header_ptr, const json &schema, + const std::vector &index_fields, std::vector &blob_fields, + uint64_t &schema_id) { + RETURN_UNEXPECTED_IF_NULL(header_ptr); + auto schema_ptr = Schema::Build("mindrecord", schema); + CHECK_FAIL_RETURN_UNEXPECTED(schema_ptr != nullptr, "[Internal ERROR] Failed to build schema: " + schema.dump() + + "." + "Check the [ERROR] logs before for more details."); + schema_id = (*header_ptr)->AddSchema(schema_ptr); + // create index创建索引 + std::vector> id_index_fields; + if (!index_fields.empty()) { + (void)transform(index_fields.begin(), index_fields.end(), std::back_inserter(id_index_fields), + [schema_id](const std::string &el) { return std::make_pair(schema_id, el); }); + RETURN_IF_NOT_OK((*header_ptr)->AddIndexFields(id_index_fields)); + } + + auto build_schema_ptr = (*header_ptr)->GetSchemas()[0]; + blob_fields = build_schema_ptr->GetBlobFields(); + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 5cb42494bff3264741adabcb20fbb028cc5f8b20 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:03:07 +0800 Subject: [PATCH 126/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_index.cc | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_index.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_index.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_index.cc new file mode 100644 index 00000000000..a0ae28385f6 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_index.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_index.h"//按照路径寻找以下文件，导入到本文件 + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +// table name for index索引的表名 +const char TABLENAME[] = "index_table"; +//调用Index类中Index函数给成员变量database_name_、table_name_赋值 +Index::Index() : database_name_(""), table_name_(TABLENAME) {} +//创建Index空间下void型AddIndexField函数,调用fields_类中emplace_back函数 +void Index::AddIndexField(const int64_t &schemaId, const std::string &field) { + fields_.emplace_back(pair(schemaId, field)); +} + +// Get attribute list获取属性列表 +std::vector> Index::GetFields() { return fields_; } +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 3e1066e79fc534059e72e4d7a0d7e611e148b1fb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:03:21 +0800 Subject: [PATCH 127/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_page.cc | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_page.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_page.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_page.cc new file mode 100644 index 00000000000..fff71065615 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_page.cc @@ -0,0 +1,58 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_page.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "pybind11/pybind11.h" + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//调用json Page类下GetPage函数,进行变量初始化 +//判断函数row_group_ids_.size()的返回值是否为0，若是则将row_groups数组中"id"和"offset"的值设为0，若不是则分别设为rg.first和rg.second +//返回str_page +json Page::GetPage() const { + json str_page; + str_page["page_id"] = page_id_; + str_page["shard_id"] = shard_id_; + str_page["page_type"] = page_type_; + str_page["page_type_id"] = page_type_id_; + str_page["start_row_id"] = start_row_id_; + str_page["end_row_id"] = end_row_id_; + if (row_group_ids_.size() == 0) { + json row_groups = json({}); + row_groups["id"] = 0; + row_groups["offset"] = 0; + str_page["row_group_ids"].push_back(row_groups); + } else { + for (const auto &rg : row_group_ids_) { + json row_groups = json({}); + row_groups["id"] = rg.first; + row_groups["offset"] = rg.second; + str_page["row_group_ids"].push_back(row_groups); + } + } + str_page["page_size"] = page_size_; + return str_page; +} +//调用Page类下void型DeleteLastGroupId函数 +//判断row_group_ids_.empty()函数的返回值是否为0，若是则修改page_size_的值为row_group_ids_.back().second，调用row_group_ids_.pop_back函数 +void Page::DeleteLastGroupId() { + if (!row_group_ids_.empty()) { + page_size_ = row_group_ids_.back().second; + row_group_ids_.pop_back(); + } +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 8d770d703dd0d9ee35abade89b246b72e70ad822 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:03:36 +0800 Subject: [PATCH 128/160] ADD file via upload --- .../mindrecord/meta/shard_pk_sample.cc | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_pk_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_pk_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_pk_sample.cc new file mode 100644 index 00000000000..7f452b9b46c --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_pk_sample.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_pk_sample.h"//按照路径寻找以下文件，导入到本文件 + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardPkSample空间中ShardPkSample函数,输入三种参数调用指定函数 +//对参数ShardCategory、shuffle_、num_samples_进行进行参数初始化 +ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples) + : ShardCategory(category_field, num_elements, std::numeric_limits::max(), true), + shuffle_(false), + num_samples_(num_samples) {} +//引用ShardPkSample空间中ShardPkSample函数,输入四种参数调用指定函数 +//对参数ShardCategory进行进行参数初始化 +ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, + int64_t num_samples) + : ShardCategory(category_field, num_elements, num_categories, true), shuffle_(false), num_samples_(num_samples) {} +//引用ShardPkSample空间中ShardPkSample函数,输入四种参数调用指定函数 +//对参数ShardCategory、shuffle_op_进行进行参数初始化 +ShardPkSample::ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, + uint32_t seed, int64_t num_samples) + : ShardCategory(category_field, num_elements, num_categories, true), shuffle_(true), num_samples_(num_samples) { + shuffle_op_ = std::make_shared(seed, kShuffleSample); // 进行重新排序和替换 +} +//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量 +//判断shuffle_是否为真,若是则调用RETURN_IF_NOT_OK函数 +//返回Status中Ok函数的返回值 +Status ShardPkSample::SufExecute(ShardTaskList &tasks) { + if (shuffle_ == true) { + RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 36359a96fb488b9f3ba68aaec0944b4f49111330 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:03:52 +0800 Subject: [PATCH 129/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_sample.cc | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sample.cc new file mode 100644 index 00000000000..47e9148b53b --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sample.cc @@ -0,0 +1,203 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_sample.h"//按照路径寻找以下文件，导入到本文件 + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardSample空间中ShardSample函数,输入参数n调用指定函数 +//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化 +ShardSample::ShardSample(int64_t n) + : numerator_(0), + denominator_(0), + partition_id_(0), + no_of_samples_(n), + indices_({}), + sampler_type_(kCustomTopNSampler), + offset_(-1) {} +//引用ShardSample空间中ShardSample函数,输入两种参数调用指定函数 +//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化 +ShardSample::ShardSample(int64_t num, int64_t den) + : numerator_(num), + denominator_(den), + partition_id_(0), + no_of_samples_(0), + indices_({}), + sampler_type_(kCustomTopPercentSampler), + offset_(-1) {} +//引用ShardSample空间中ShardSample函数,输入五种参数调用指定函数 +//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化 +ShardSample::ShardSample(int64_t num, int64_t den, int64_t par, int64_t no_of_samples, int64_t offset) + : numerator_(num), + denominator_(den), + partition_id_(par), + no_of_samples_(no_of_samples), + indices_({}), + sampler_type_(kCustomTopPercentSampler), + offset_(offset) {} +//引用ShardSample空间中ShardSample函数,输入参数indices调用指定函数 +//对参数numerator_、denominator_、partition_id_、no_of_samples_、indices_、sampler_type_、offset_进行参数初始化 +ShardSample::ShardSample(const std::vector &indices) + : numerator_(0), + denominator_(0), + partition_id_(0), + no_of_samples_(0), + indices_(indices), + sampler_type_(kSubsetSampler) {} +//引用ShardSample空间中ShardSample函数,输入参数indices调用指定函数，此函数继承ShardSample函数 +//对参数sampler_type_、shuffle_op_进行参数初始化 +ShardSample::ShardSample(const std::vector &indices, uint32_t seed) : ShardSample(indices) { + sampler_type_ = kSubsetRandomSampler; + shuffle_op_ = std::make_shared(seed); +} +//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为int64_t型参数 +//判断sampler_type_与kCustomTopNSampler是否赋值相同，若是则返回no_of_samples_ +//判断sampler_type_与kCustomTopPercentSampler是否复制相同，若是则再次判断dataset_size与denominator_的模是否为0，若是则返回dataset_size与denominator_和numerator_的乘积的商 +//若不是则返回dataset_size与denominator_和numerator_的乘积的商+1的值 +//判断sampler_type_与kSubsetRandomSampler的值或者sampler_type_与kSubsetSampler的值是否相等，若任一等式成立则返回indices_.size函数的返回值 +//若均不符合条件，最后返回0 +int64_t ShardSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (sampler_type_ == kCustomTopNSampler) { + return no_of_samples_; + } + + if (sampler_type_ == kCustomTopPercentSampler) { + if (dataset_size % denominator_ == 0) { + return dataset_size / denominator_ * numerator_; + } else { + return dataset_size / denominator_ * numerator_ + 1; + } + } + if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { + return indices_.size(); + } + return 0; +} +//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量 +//判断tasks.permutation_.empty函数的返回值是否为真，若是则将tasks.sample_ids_.size()的返回值赋给total_no并调用CHECK_FAIL_RETURN_UNEXPECTED函数 +//判断sampler_type_与kSubsetRandomSampler的值或者sampler_type_与kSubsetSampler的值是否相等，若任一等式成立则进入循环，修改index的赋值并调用new_tasks.AssignTask函数 +//若两等式均不成立，则判断nums_per_shard_.empty函数的返回值是否为真，若是则进入循环，调用new_tasks.AssignTask函数并累加count变量，直至no_of_samples_不等于0且count与no_of_samples_相等均成立时跳出循环 +//若不是，则进入另一循环，循环次数与上式不同，循环运行内容相同 +//若tasks.permutation_.empty函数的返回值为假，则将tasks.sample_ids_.size()的返回值赋给total_no并调用CHECK_FAIL_RETURN_UNEXPECTED函数，直接进入执行相同操作的循环，次数不同 +//在输出返回值前调用ShardTaskList类中的TaskListSwap函数 +//返回Status中Ok函数的返回值 +Status ShardSample::UpdateTasks(ShardTaskList &tasks, int64_t taking) { + if (tasks.permutation_.empty()) { + ShardTaskList new_tasks; + auto total_no = tasks.sample_ids_.size(); + CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0, + "[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no)); + if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { + for (int64_t i = 0; i < indices_.size(); ++i) { + int64_t index = ((indices_[i] % total_no) + total_no) % total_no; + new_tasks.AssignTask(tasks, index); // different mod result between c and python c和python有不同mod结果 + } + } else { + int64_t count = 0; + if (nums_per_shard_.empty()) { + for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { + if (no_of_samples_ != 0 && count == no_of_samples_) break; + new_tasks.AssignTask(tasks, i % total_no); // rounding up. if overflow, go back to start四舍五入。如果溢出，返回开始 + count++; + } + } else { + // Get samples within a specific range获取特定范围内的样本 + int64_t i = partition_id_ - 1 >= 0 ? nums_per_shard_[partition_id_ - 1] : 0; + for (; i < nums_per_shard_[partition_id_]; i++) { + if (no_of_samples_ != 0 && count == no_of_samples_) break; + new_tasks.AssignTask(tasks, i % total_no); + count++; + } + } + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } else { + ShardTaskList new_tasks; + int64_t total_no = tasks.permutation_.size(); + CHECK_FAIL_RETURN_UNEXPECTED(total_no > 0, + "[Internal ERROR] 'total_no' should be positive but got: " + std::to_string(total_no)); + int64_t cnt = 0; + for (int64_t i = partition_id_ * taking; i < (partition_id_ + 1) * taking; i++) { + if (no_of_samples_ != 0 && cnt == no_of_samples_) break; + new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]); + cnt++; + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } + return Status::OK(); +} +//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量 +//判断offset_是否为-1，若不是则进入循环，对samples_per_buffer_、remainder进行赋值并对remainder、offset_进行判断后调整samples_per_buffer_的赋值，最终调用nums_per_shard_.push_back函数 +//判断sampler_type_与kCustomTopNSampler是否相等，若相等则对no_of_samples_、taking = no_of_samples_赋值 +//若不相等则判断sampler_type_ 与kSubsetRandomSampler是否相等或sampler_type_与kSubsetSampler是否相等，其中任一成立即调用CHECK_FAIL_RETURN_UNEXPECTED函数并输出非法输入警告 +//若均不符合则判断numerator_、denominator_是否均大于0且numerator_是否小于等于denominator_ ，若是则继续判断numerator_是否等于1且denominator_是否大于1，若是则对taking赋值 +//若不是则对taking进行其他操作的赋值 +//若不符合第一条件，则调用RETURN_STATUS_UNEXPECTED函数输出标准变量不符合警告 +//返回UpdateTasks函数的返回值 +Status ShardSample::Execute(ShardTaskList &tasks) { + if (offset_ != -1) { + int64_t old_v = 0; + int64_t num_rows_ = tasks.sample_ids_.size(); + for (int64_t x = 0; x < denominator_; x++) { + int64_t samples_per_buffer_ = (num_rows_ + offset_) / denominator_; + int64_t remainder = (num_rows_ + offset_) % denominator_; + if (x < remainder) samples_per_buffer_++; + if (x < offset_) samples_per_buffer_--; + old_v += samples_per_buffer_; + // nums_per_shard_ is used to save the current shard's ending index nums_per_shard用于保存当前碎片的结束索引 + nums_per_shard_.push_back(old_v); + } + } + int no_of_categories = static_cast(tasks.categories); + int64_t total_no = tasks.sample_ids_.size(); + int64_t taking = 0; + if (sampler_type_ == kCustomTopNSampler) { // non sharding case constructor #1非分片case构造函数#1 + no_of_samples_ = std::min(no_of_samples_, total_no); + taking = no_of_samples_ - no_of_samples_ % no_of_categories; + } else if (sampler_type_ == kSubsetRandomSampler || sampler_type_ == kSubsetSampler) { + CHECK_FAIL_RETURN_UNEXPECTED(static_cast(indices_.size()) <= total_no, + "Invalid input, indices size: " + std::to_string(indices_.size()) + + " should be less than or equal to database size: " + std::to_string(total_no) + "."); + } else { // constructor TopPercent顶部百分比 + if (numerator_ > 0 && denominator_ > 0 && numerator_ <= denominator_) { + if (numerator_ == 1 && denominator_ > 1) { // sharding分片 + taking = (total_no + denominator_ - 1) / denominator_; + } else { // non sharding不分片 + taking = total_no * numerator_ / denominator_; + taking -= (taking % no_of_categories); + } + } else { + RETURN_STATUS_UNEXPECTED("[Internal ERROR] 'numerator_': " + std::to_string(numerator_) + + " should be positive and less than denominator_: " + std::to_string(denominator_) + "."); + } + } + return UpdateTasks(tasks, taking); +} +//在ShardDistributedSample空间中创建Status型PreExecute函数,返回值为Status变量 +//判断sampler_type_和kSubsetRandomSampler的值是否相等，若是则调用RETURN_IF_NOT_OK函数 +//返回Status中Ok函数的返回值 +Status ShardSample::SufExecute(ShardTaskList &tasks) { + if (sampler_type_ == kSubsetRandomSampler) { + RETURN_IF_NOT_OK((*shuffle_op_)(tasks)); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 5faf38830e326ef1c3ea5f11efc981c5763ae39b Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:04:11 +0800 Subject: [PATCH 130/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_schema.cc | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_schema.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_schema.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_schema.cc new file mode 100644 index 00000000000..575ba863350 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_schema.cc @@ -0,0 +1,179 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_schema.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "utils/ms_utils.h" + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//建立存储空间的函数 +//引用Schema空间中的Build函数，返回空间指针 +std::shared_ptr Schema::Build(std::string desc, const json &schema) { + // validate check验证检查 + if (!Validate(schema)) { + return nullptr; + } + + std::vector blob_fields = PopulateBlobFields(schema); + Schema object_schema; + object_schema.desc_ = std::move(desc); + object_schema.blob_fields_ = std::move(blob_fields); + object_schema.schema_ = schema; + object_schema.schema_id_ = -1; + return std::make_shared(object_schema); +} +//引用Schema空间中的GetDesc函数，返回desc_参数 +std::string Schema::GetDesc() const { return desc_; } +//引用Schema空间中GetSchema函数，返回str_schema数组 +//str_schema数组中储存这Schema的基本数值 +json Schema::GetSchema() const { + json str_schema; + str_schema["desc"] = desc_; + str_schema["schema"] = schema_; + str_schema["blob_fields"] = blob_fields_; + return str_schema; +} +//引用Schema空间中SetSchemaID函数 +//使schema_id_储存相应的id数值 +void Schema::SetSchemaID(int64_t id) { schema_id_ = id; } +//引用Schema空间中GetSchemaID函数 +//获得schema_id_的值 +int64_t Schema::GetSchemaID() const { return schema_id_; } +//引用Schema空间中GetGetBlobFields函数 +//获得blob_fields_的值 +std::vector Schema::GetBlobFields() const { return blob_fields_; } +//引用Schema空间中的PopulateBlobFields函数 +//依次对比schema数组中各项的size、shape、type是否符合要求，若符合则将该项的key存储进blob_fields +//最终返回blob_fields +std::vector Schema::PopulateBlobFields(json schema) { + std::vector blob_fields; + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + json it_value = it.value(); + if ((it_value.size() == kInt2 && it_value.find("shape") != it_value.end()) || it_value["type"] == "bytes") { + blob_fields.emplace_back(it.key()); + } + } + return blob_fields; +} +//引用Schema空间中的ValidateNumberShape函数 +bool Schema::ValidateNumberShape(const json &it_value) { + //判断传入的数据是否是数组的结尾，若是则输出错误信息，并返回false + if (it_value.find("shape") == it_value.end()) { + MS_LOG(ERROR) << "Invalid schema, 'shape' object can not found in " << it_value.dump() + << ". Please check the input schema."; + return false; + } + //给shape变量赋值，值为输入数据的类型 + //判断其值是否在标准范围内，若不是则输出错误信息，并返回false + auto shape = it_value["shape"]; + if (!shape.is_array()) { + MS_LOG(ERROR) << "Invalid schema, the value of 'shape' should be list format but got: " << it_value["shape"] + << ". Please check the input schema."; + return false; + } + //给num_negtive_one变量赋值为0 + //进入循环并不断给i进行赋值，值为不同的shape + //若出现i等于0或i小于-1的情况，输出错误信息，并返回false + //若出现i等于-1的情况，则num_negtive_one+1进行计数 + int num_negtive_one = 0; + for (const auto &i : shape) { + if (i == 0 || i < -1) { + MS_LOG(ERROR) << "Invalid schema, the element of 'shape' value should be -1 or greater than 0 but got: " << i + << ". Please check the input schema."; + return false; + } + if (i == -1) { + num_negtive_one++; + } + } + //判断num_negtive_one是否大于1，若是则输出错误信息，并返回false + if (num_negtive_one > 1) { + MS_LOG(ERROR) << "Invalid schema, only 1 variable dimension(-1) allowed in 'shape' value but got: " + << it_value["shape"] << ". Please check the input schema."; + return false; + } + + return true; +} +////引用Schema空间中的Validate函数 +bool Schema::Validate(json schema) { + //判断schema空间是否为空，若是则输出错误信息，并返回false + if (schema.empty()) { + MS_LOG(ERROR) << "Invalid schema, schema is empty. Please check the input schema."; + return false; + } + //进入循环，依次判断schema空间中所有数据是否符合标准，若不是则输出错误信息，并返回false + for (json::iterator it = schema.begin(); it != schema.end(); ++it) { + // make sure schema key name must be composed of '0-9' or 'a-z' or 'A-Z' or '_'确保架构密钥名称必须由“0-9”、“a-z”、“a-z”或“_”组成 + if (!ValidateFieldName(it.key())) { + MS_LOG(ERROR) << "Invalid schema, field name: " << it.key() + << "is not composed of '0-9' or 'a-z' or 'A-Z' or '_'. Please rename the field name in schema."; + return false; + } + //确保数据的type存在 + json it_value = it.value(); + if (it_value.find("type") == it_value.end()) { + MS_LOG(ERROR) << "Invalid schema, 'type' object can not found in field " << it_value.dump() + << ". Please add the 'type' object for field in schema."; + return false; + } + //确保数据的type合法 + if (kFieldTypeSet.find(it_value["type"]) == kFieldTypeSet.end()) { + MS_LOG(ERROR) << "Invalid schema, the value of 'type': " << it_value["type"] + << " is not supported.\nPlease modify the value of 'type' to 'int32', 'int64', 'float32', " + "'float64', 'string', 'bytes' in schema."; + return false; + } + //确保数据的合法数据运行后报错 + if (it_value.size() == kInt1) { + continue; + } + //确保schema空间中存储空间为合法空间 + if (it_value["type"] == "bytes" || it_value["type"] == "string") { + MS_LOG(ERROR) + << "Invalid schema, no other field can be added when the value of 'type' is 'string' or 'types' but got: " + << it_value.dump() << ". Please remove other fields in schema."; + return false; + } + //确保schema空间中存储空间的type和shape属性完整 + if (it_value.size() != kInt2) { + MS_LOG(ERROR) << "Invalid schema, the fields should be 'type' or 'type' and 'shape' but got: " << it_value.dump() + << ". Please check the schema."; + return false; + } + //确保数据的shape属性符合条件 + if (!ValidateNumberShape(it_value)) { + return false; + } + } + + return true; +} +//重载mindrecord空间中的Schema中的b +//加入判断：判断此空间下GetDesc函数和GetSchema函数的返回值是否与b的相应函数的返回值相同 +//任意等式不成立，则返回false，反之返回true +bool Schema::operator==(const mindrecord::Schema &b) const { + if (this->GetDesc() != b.GetDesc() || this->GetSchema() != b.GetSchema()) { + return false; + } + return true; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 85becf4d316d4927b0cb224a6434836e43010f55 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:04:30 +0800 Subject: [PATCH 131/160] ADD file via upload --- .../meta/shard_sequential_sample.cc | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sequential_sample.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sequential_sample.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sequential_sample.cc new file mode 100644 index 00000000000..eba5daf5c3c --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_sequential_sample.cc @@ -0,0 +1,81 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_sequential_sample.h"//按照路径寻找以下文件，导入到本文件 + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardSequentialSample空间中ShardSequentialSample函数,输入两种参数调用指定函数 +//对参数ShardSample、offset_、per_、per_offset_进行参数初始化 +ShardSequentialSample::ShardSequentialSample(int64_t n, int64_t offset) + : ShardSample(n), offset_(offset), per_(0.0f), per_offset_(0.0f) {} +//引用ShardSequentialSample空间中ShardSequentialSample函数,输入两种参数调用指定函数 +//对参数ShardSample、offset_、per_、per_offset_进行参数初始化 +ShardSequentialSample::ShardSequentialSample(float per, float per_offset) + : ShardSample(0), offset_(0), per_(per), per_offset_(per_offset) {} +//引用ShardSequentialSample空间中GetNumSamples函数 +//判断no_of_samples_是否等于0且per_是否在-kEpsilon到kEpsilon区间内，若是则返回dataset_size +//判断per_是否在kEpsilon到1.0f区间内，若是则返回dataset_size与kEpsilon的乘积 +//若均不符合则返回dataset_size和no_of_samples_之间较小的值 +int64_t ShardSequentialSample::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) { + return dataset_size; + } + if (per_ > kEpsilon && per_ <= 1.0f) { + return dataset_size * kEpsilon; + } + return std::min(static_cast(no_of_samples_), dataset_size); +} +//引用ShardSequentialSample空间中Execute函数 +Status ShardSequentialSample::Execute(ShardTaskList &tasks) { + int64_t taking; + int64_t total_no = static_cast(tasks.sample_ids_.size()); + //判断no_of_samples_是否等于0且per_是否在-kEpsilon到kEpsilon区间内，若是则给taking赋值，值为total_no + //判断per_是否在kEpsilon到1.0f区间内，若是则给taking赋值，值为total_no与kEpsilon的乘积 + //若均不符合则返回total_no和no_of_samples_之间较小的值 + if (no_of_samples_ == 0 && (per_ >= -kEpsilon && per_ <= kEpsilon)) { + taking = total_no; + } else if (per_ > kEpsilon && per_ <= 1.0f) { + taking = total_no * kEpsilon; + } else { + taking = std::min(static_cast(no_of_samples_), total_no); + } + //判断tasks中permutation_是否为空，若是则给total_no赋值，值为tasks.Size()，并进入循环，依次给new_tasks赋值 + //引用ShardTaskList空间中的TaskListSwap函数 + if (tasks.permutation_.empty()) { + ShardTaskList new_tasks; + total_no = static_cast(tasks.Size()); + for (int64_t i = offset_; i < taking + offset_; ++i) { + new_tasks.AssignTask(tasks, i % total_no); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } else { // shuffled洗牌 + ShardTaskList new_tasks; + total_no = static_cast(tasks.permutation_.size()); + for (int64_t i = offset_; i < taking + offset_; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i % total_no]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } + return Status::OK(); +} + +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 9a69ff2fdb0492aa373ed45e50b1e221fc91d0d9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:04:44 +0800 Subject: [PATCH 132/160] ADD file via upload --- .../minddata/mindrecord/meta/shard_shuffle.cc | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc new file mode 100644 index 00000000000..68d04ceeb17 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc @@ -0,0 +1,215 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_shuffle.h"//按照路径寻找以下文件，导入到本文件 + +#include // + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardShuffle空间中ShardShuffle函数,输入两种参数调用指定函数 +//对参数shuffle_seed_、no_of_samples_、replacement_、reshuffle_each_epoch_、shuffle_type_进行进行参数初始化 +ShardShuffle::ShardShuffle(uint32_t seed, ShuffleType shuffle_type) + : shuffle_seed_(seed), + no_of_samples_(0), + replacement_(false), + reshuffle_each_epoch_(true), + shuffle_type_(shuffle_type) {} +//引用ShardShuffle空间中ShardShuffle函数,输入五种参数调用指定函数 +//对参数shuffle_seed_、no_of_samples_、replacement_、reshuffle_each_epoch_、shuffle_type_进行进行参数初始化 +ShardShuffle::ShardShuffle(uint32_t seed, int64_t no_of_samples, bool replacement, bool reshuffle_each_epoch, + ShuffleType shuffle_type) + : shuffle_seed_(seed), + no_of_samples_(no_of_samples), + replacement_(replacement), + reshuffle_each_epoch_(reshuffle_each_epoch), + shuffle_type_(shuffle_type) {} +//在ShardShuffle空间中创建GetNumSamples函数 +//判断replacement_是否为真，若是则返回dataset_size或no_of_samples_ +int64_t ShardShuffle::GetNumSamples(int64_t dataset_size, int64_t num_classes) { + if (replacement_) { + return no_of_samples_ == 0 ? dataset_size : no_of_samples_; + } + return no_of_samples_ == 0 ? dataset_size : std::min(dataset_size, no_of_samples_); +} +//在ShardShuffle空间中创建CategoryShuffle函数 +Status ShardShuffle::CategoryShuffle(ShardTaskList &tasks) { + int64_t individual_size = tasks.sample_ids_.size() / tasks.categories; + std::vector> new_permutations(tasks.categories, std::vector(individual_size)); + //进入循环，反复调用shuffle + for (int64_t i = 0; i < tasks.categories; i++) { + for (int64_t j = 0; j < individual_size; j++) new_permutations[i][j] = j; + std::shuffle(new_permutations[i].begin(), new_permutations[i].end(), std::default_random_engine(shuffle_seed_)); + } + tasks.permutation_.clear(); + //进入循环，创建tasks.permutation_列表 + for (int64_t j = 0; j < individual_size; j++) { + for (int64_t i = 0; i < tasks.categories; i++) { + tasks.permutation_.push_back(new_permutations[i][j] * tasks.categories + i); + } + } + //进入循环，遍历new_tasks列表 + ShardTaskList new_tasks; + for (int64_t i = 0; i < individual_size; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + + return Status::OK(); +} +//在ShardShuffle空间中创建ShuffleFiles函数 +Status ShardShuffle::ShuffleFiles(ShardTaskList &tasks) { + //判断no_of_samples_是否为0，若是则赋值tasks的大小 + if (no_of_samples_ == 0) { + no_of_samples_ = tasks.Size(); + } + CHECK_FAIL_RETURN_UNEXPECTED( + no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_)); + auto shard_sample_cout = GetShardSampleCount(); + + // shuffle the files index洗牌文件索引 + std::vector shuffle_files; + for (int64_t i = 0; i < shard_sample_cout.size(); i++) { + shuffle_files.push_back(i); + } + std::shuffle(shuffle_files.begin(), shuffle_files.end(), std::default_random_engine(shuffle_seed_)); + + // reconstruct the permutation between files重建文件之间的排列 + // -- before -- + // file1: [0, 1, 2] + // file2: [3, 4, 5, 6] + // file3: [7, 8] + // file4: [9, 10] + // files: [file1, file2, file3, file4] + // permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + // -- after -- + // files: [file4, file1, file3, file2] + // permutation : [9, 10, 0, 1, 2, 7, 8, 3, 4, 5, 6] + auto original_permutation = tasks.permutation_; + int64_t whole_index = 0; + //进入循环，遍历shuffle_files列表并进行复制 + for (int64_t i = 0; i < shuffle_files.size(); i++) { + int64_t start_index = 0; + int64_t current_size = 0; + if (shuffle_files[i] == 0) { + start_index = 0; + current_size = shard_sample_cout[shuffle_files[i]]; + } else { + start_index = shard_sample_cout[shuffle_files[i] - 1]; + current_size = shard_sample_cout[shuffle_files[i]] - start_index; + } + std::copy(original_permutation.begin() + start_index, original_permutation.begin() + start_index + current_size, + tasks.permutation_.begin() + whole_index); + whole_index += current_size; + } + //进入循环，遍历new_tasks列表 + auto total_no = tasks.Size(); + int64_t samples_to_assign = + (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); + ShardTaskList new_tasks; + for (int64_t i = 0; i < samples_to_assign; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + return Status::OK(); +} +//在ShardShuffle空间中创建ShuffleInFiles函数 +Status ShardShuffle::ShuffleInfile(ShardTaskList &tasks) { + //判断no_of_samples_是否为0，若是则赋值tasks的大小 + if (no_of_samples_ == 0) { + no_of_samples_ = tasks.Size(); + } + CHECK_FAIL_RETURN_UNEXPECTED( + no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + std::to_string(no_of_samples_)); + // reconstruct the permutation in file重建文件中的排列 + // -- before -- + // file1: [0, 1, 2] + // file2: [3, 4, 5, 6] + // file3: [7, 8] + // file4: [9, 10] + // files: [file1, file2, file3, file4] + // permutation: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + // -- after -- + // permutation: [2, 0, 1, 4, 6, 3, 5, 8, 7, 9, 10] + auto shard_sample_cout = GetShardSampleCount(); + int64_t start_index = 0; + //进入循环，计算出start_index的值 + for (int64_t i = 0; i < shard_sample_cout.size(); i++) { + auto current_size = shard_sample_cout[i] - start_index; + std::shuffle(tasks.permutation_.begin() + start_index, tasks.permutation_.begin() + start_index + current_size, + std::default_random_engine(shuffle_seed_)); + start_index = shard_sample_cout[i]; + } + auto total_no = tasks.Size(); + ShardTaskList new_tasks; + int64_t samples_to_assign = + (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); + //进入循环，遍历new_tasks列表 + for (int64_t i = 0; i < samples_to_assign; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + return Status::OK(); +} +//在ShardShuffle空间中创建Execute函数 +Status ShardShuffle::Execute(ShardTaskList &tasks) { + // + if (reshuffle_each_epoch_) { + shuffle_seed_++; + } + CHECK_FAIL_RETURN_UNEXPECTED(tasks.categories >= 1, + "[Internal ERROR] task categories should be greater than or equal to 1 but got: " + + std::to_string(tasks.categories)); + if (shuffle_type_ == kShuffleSample) { // shuffle each sample洗牌每个样品 + if (tasks.permutation_.empty() == true) { + tasks.MakePerm(); + } + if (GetShuffleMode() == dataset::ShuffleMode::kGlobal) { + if (replacement_ == true) { + ShardTaskList new_tasks; + if (no_of_samples_ == 0) { + no_of_samples_ = tasks.sample_ids_.size(); + } + CHECK_FAIL_RETURN_UNEXPECTED(no_of_samples_ > 0, "Invalid input, 'num_samples' should be positive but got: " + + std::to_string(no_of_samples_)); + for (uint32_t i = 0; i < no_of_samples_; ++i) { + new_tasks.AssignTask(tasks, tasks.GetRandomTaskID()); + } + + ShardTaskList::TaskListSwap(tasks, new_tasks); + } else { + std::shuffle(tasks.permutation_.begin(), tasks.permutation_.end(), std::default_random_engine(shuffle_seed_)); + auto total_no = tasks.Size(); + ShardTaskList new_tasks; + int64_t samples_to_assign = + (no_of_samples_ > 0 && no_of_samples_ < total_no) ? no_of_samples_ : tasks.sample_ids_.size(); + for (int64_t i = 0; i < samples_to_assign; ++i) { + new_tasks.AssignTask(tasks, tasks.permutation_[i]); + } + ShardTaskList::TaskListSwap(tasks, new_tasks); + } + } else if (GetShuffleMode() == dataset::ShuffleMode::kInfile) { + RETURN_IF_NOT_OK(ShuffleInfile(tasks)); + } else if (GetShuffleMode() == dataset::ShuffleMode::kFiles) { + RETURN_IF_NOT_OK(ShuffleFiles(tasks)); + } + } else { // shuffle unit like: (a1, b1, c1),(a2, b2, c2),..., (an, bn, cn)分组洗牌 + return this->CategoryShuffle(tasks); + } + return Status::OK(); +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 55c7621fbfbe06154a421cc7bea5b11ea7e6b604 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:04:58 +0800 Subject: [PATCH 133/160] ADD file via upload --- .../mindrecord/meta/shard_statistics.cc | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_statistics.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_statistics.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_statistics.cc new file mode 100644 index 00000000000..89489f8aa79 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_statistics.cc @@ -0,0 +1,109 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/mindrecord/include/shard_statistics.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "pybind11/pybind11.h" + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::ERROR;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//建立存储空间的函数 +//引用Schema空间中的Build函数，返回空间指针 +std::shared_ptr Statistics::Build(std::string desc, const json &statistics) { + // validate check验证检查 + if (!Validate(statistics)) { + return nullptr; + } + Statistics object_statistics; + object_statistics.desc_ = std::move(desc); + object_statistics.statistics_ = statistics; + object_statistics.statistics_id_ = -1; + return std::make_shared(object_statistics); +} +//引用Statistics空间中的GetDesc函数，返回desc_参数 +std::string Statistics::GetDesc() const { return desc_; } +//引用Statistics空间中GetStatistics函数，返回str_schema数组 +//str_statistics数组中储存这statistics的基本数值 +json Statistics::GetStatistics() const { + json str_statistics; + str_statistics["desc"] = desc_; + str_statistics["statistics"] = statistics_; + return str_statistics; +} +//引用Statistics空间中SetStatisticsID函数 +//使statistics_id_储存相应的id数值 +void Statistics::SetStatisticsID(int64_t id) { statistics_id_ = id; } +//引用Statistics空间中GetStatisticsID函数 +//获得statistics_id_的值 +int64_t Statistics::GetStatisticsID() const { return statistics_id_; } +//引用Statistics空间中的Validate函数 +bool Statistics::Validate(const json &statistics) { + //判断数据是否符合标准，若不符则输出错误信息，并返回false + if (statistics.size() != kInt1) { + MS_LOG(ERROR) << "Invalid data, 'statistics' is empty."; + return false; + } + //判断数据是否是statistics空间的结尾，若是则输出错误信息，并返回false + if (statistics.find("level") == statistics.end()) { + MS_LOG(ERROR) << "Invalid data, 'level' object can not found in statistic"; + return false; + } + //若以上都不符合则返回LevelRecursive函数 + return LevelRecursive(statistics["level"]); +} +//引用Statistics空间中的LevelRecursive函数 +bool Statistics::LevelRecursive(json level) { + bool ini = true; + //进入循环，遍历level数组 + for (json::iterator it = level.begin(); it != level.end(); ++it) { + json a = it.value(); + //判断该空间大小是否符合标准2，若符合则判断该空间的key和count是否与无数据空间一致，若是则返回错误信息并返回false + //若空间大小不符合标准2，则判断空间大小是否符合标准3，若符合则判断该空间key、count、level是否与无数据空间一致，若是则返回错误信息并返回false，若不符合则给ini变量赋值 + //若均不符合，则返回错误信息，返回false + //最后返回ini的值 + if (a.size() == kInt2) { + if ((a.find("key") == a.end()) || (a.find("count") == a.end())) { + MS_LOG(ERROR) << "Invalid data, the node field is 2, but 'key'/'count' object does not existed"; + return false; + } + } else if (a.size() == kInt3) { + if ((a.find("key") == a.end()) || (a.find("count") == a.end()) || a.find("level") == a.end()) { + MS_LOG(ERROR) << "Invalid data, the node field is 3, but 'key'/'count'/'level' object does not existed"; + return false; + } else { + ini = LevelRecursive(a.at("level")); + } + } else { + MS_LOG(ERROR) << "Invalid data, the node field is not equal to 2 or 3"; + return false; + } + } + return ini; +} +//重载mindrecord空间中的Statistics中的b +//加入判断：判断此空间下GetStatistics函数和GetStatistics函数的返回值是否与b的相应函数的返回值相同 +//任意等式不成立，则返回false，反之返回true +bool Statistics::operator==(const Statistics &b) const { + if (this->GetStatistics() != b.GetStatistics()) { + return false; + } + return true; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 44584f387fd6f358468e33b323f29cb02b0484b8 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:05:14 +0800 Subject: [PATCH 134/160] ADD file via upload --- .../mindrecord/meta/shard_task_list.cc | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/shard_task_list.cc diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_task_list.cc b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_task_list.cc new file mode 100644 index 00000000000..3e04b975ec1 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/meta/shard_task_list.cc @@ -0,0 +1,162 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/util/random.h"//按照路径寻找以下文件，导入到本文件,以下同理 +#include "minddata/mindrecord/include/shard_task_list.h" +#include "utils/ms_utils.h" +#include "minddata/mindrecord/include/common/shard_utils.h" + +using mindspore::LogStream;//声明mindspore空间下的LogStream +using mindspore::ExceptionType::NoExceptionType;//声明mindspore空间下ExceptionType类中的NoExceptionType +using mindspore::MsLogLevel::DEBUG;//声明mindspore空间下MsLogLevel类中的ERROR + +namespace mindspore {//创建名为mindspore的空间 +namespace mindrecord {//创建名为mindrecord的空间 +//引用ShardTaskList空间中ShardTaskList函数,不输入参数调用指定函数 +//对参数categories进行参数初始化 +ShardTaskList::ShardTaskList() : categories(1) {} +//引用ShardTaskList空间中ShardTaskList函数,输入一种参数调用指定函数 +//对参数categories、permutation_、sample_ids_、task_list_进行参数初始化 +ShardTaskList::ShardTaskList(const ShardTaskList &other) + : categories(other.categories), + permutation_(other.permutation_), + sample_ids_(other.sample_ids_), + task_list_(other.task_list_) {} + //重载ShardTaskList空间中的TaskList中的other + //交换多种值的内容 +ShardTaskList &ShardTaskList::operator=(const ShardTaskList &other) { + ShardTaskList tmp(other); + std::swap(categories, tmp.categories); + permutation_.swap(tmp.permutation_); + sample_ids_.swap(tmp.sample_ids_); + task_list_.swap(tmp.task_list_); + return *this; +} +//引用ShardTaskList空间中InitSampleIds函数 +//进入循环，创建列表 +void ShardTaskList::InitSampleIds() { + // no-op if there already exists sample ids. Do not clobber previous list如果已经存在示例id，则无操作。不要破坏上一个列表 + if (sample_ids_.empty()) { + sample_ids_ = std::vector(task_list_.size()); + for (auto i = 0; i < task_list_.size(); i++) { + sample_ids_[i] = i; + } + } +} +//引用ShardTaskList空间中MakePerm函数 +//创建permutation_列表 +void ShardTaskList::MakePerm() { + int64_t perm_size = sample_ids_.size(); + permutation_ = std::vector(perm_size); + for (int64_t i = 0; i < perm_size; i++) { + permutation_[i] = i; + } +} +//引用ShardTaskList空间中TaskListSwap函数 +// Swap the new_tasks with orig_tasks将新任务与orig_tasks交换 +void ShardTaskList::TaskListSwap(ShardTaskList &orig_tasks, ShardTaskList &new_tasks) { + // When swapping, if the orig_tasks contains fields that need to be preserved after the swap, then swapping with a交换时，如果orig_tasks包含交换后需要保留的字段，则使用 + // new_tasks that does not have those fields will result in clobbering/losing the data after the swap.没有这些字段的new_tasks将导致交换后数据丢失。 + // The task_list_ should not be lost/clobbered.task_list_不应丢失/丢失。 + // This function can be called in the middle of mindrecord's epoch, when orig_tasks.task_list_ is still being这个函数可以在mindrecord的时代中期调用，当orig_任务时。task_list_仍在 + // used by mindrecord op's worker threads. So don't touch its task_list_ since this field should be preserved anyways. + //由mindrecord op的工作线程使用。因此，不要触摸其task_list_，因为无论如何都应该保留此字段。 + std::swap(orig_tasks.categories, new_tasks.categories); + std::swap(orig_tasks.permutation_, new_tasks.permutation_); + std::swap(orig_tasks.sample_ids_, new_tasks.sample_ids_); +} +//引用ShardTaskList空间中PopBack函数 +//调用task_list_类中pop_back函数 +void ShardTaskList::PopBack() { task_list_.pop_back(); } +//引用ShardTaskList空间中Size函数 +//返回task_list_空间中size函数的返回值 +int64_t ShardTaskList::Size() const { return static_cast(task_list_.size()); } +//引用ShardTaskList空间中SizeOfRows函数 +int64_t ShardTaskList::SizeOfRows() const { + //判断task_list_的长度是否为0，若是则返回0 + if (task_list_.size() == 0) return static_cast(0); + + // 1 task is 1 page1个任务是1页 + const size_t kBlobInfoIndex = 2; + auto sum_num_rows = [](int64_t x, ShardTask y) { return x + std::get(y)[0]; }; + int64_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows); + return nRows; +} +//引用ShardTaskList空间中GetTaskByID函数 +//返回task_list_数组中id的返回值 +ShardTask &ShardTaskList::GetTaskByID(int64_t id) { return task_list_[id]; } +//引用ShardTaskList空间中GetTaskSampleByID函数 +//返回sample_ids_数组中id的返回值 +int64_t ShardTaskList::GetTaskSampleByID(int64_t id) { return sample_ids_[id]; } +//引用ShardTaskList空间中GetRandomTaskID函数 +int64_t ShardTaskList::GetRandomTaskID() { + std::mt19937 gen = mindspore::dataset::GetRandomDevice(); + std::uniform_int_distribution<> dis(0, sample_ids_.size() - 1); + return dis(gen); +} +//引用ShardTaskList空间中GetRandomTask函数 +ShardTask &ShardTaskList::GetRandomTask() { + std::mt19937 gen = mindspore::dataset::GetRandomDevice(); + std::uniform_int_distribution<> dis(0, task_list_.size() - 1); + return task_list_[dis(gen)]; +} +//引用ShardTaskList空间中Combine函数 +ShardTaskList ShardTaskList::Combine(std::vector &category_tasks, bool replacement, int64_t num_elements, + int64_t num_samples) { + ShardTaskList res; + //判断category_tasks是否为空，若是则返回res的值 + if (category_tasks.empty()) return res; + auto total_categories = category_tasks.size(); + res.categories = static_cast(total_categories); + //判断resplacement是否为false，若是则判断category_tasks列表中的最小值 + //在0至category_tasks最小值的区间内反复调用InsertTask函数进行数据处理，直到num_samples等于0且count等于num_samples + //若resplacement不为0，则判断category_tasks列表中的最大值 + //在0至category_tasks最小值的区间内反复调用InsertTask函数进行数据处理，直到num_samples等于0且count等于num_samples + if (replacement == false) { + auto minTasks = category_tasks[0].Size(); + for (int64_t i = 1; i < total_categories; i++) { + minTasks = std::min(minTasks, category_tasks[i].Size()); + } + int64_t count = 0; + for (int64_t task_no = 0; task_no < minTasks; task_no++) { + for (int64_t i = 0; i < total_categories; i++) { + if (num_samples != 0 && count == num_samples) break; + res.InsertTask(std::move(category_tasks[i].GetTaskByID(task_no))); + count++; + } + } + } else { + auto maxTasks = category_tasks[0].Size(); + for (int64_t i = 1; i < total_categories; i++) { + maxTasks = std::max(maxTasks, category_tasks[i].Size()); + } + if (num_elements != std::numeric_limits::max()) { + maxTasks = static_cast(num_elements); + } + int64_t count = 0; + for (int64_t i = 0; i < total_categories; i++) { + for (int64_t j = 0; j < maxTasks; j++) { + if (num_samples != 0 && count == num_samples) break; + res.InsertTask(category_tasks[i].GetRandomTask()); + count++; + } + } + } + //返回res的值 + return res; +} +} // namespace mindrecord +} // namespace mindspore -- 2.34.1 From 5f5d533920d008315a656c5440ede7d74d18fafb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:05:30 +0800 Subject: [PATCH 135/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/meta/int' --- .../ccsrc/minddata/mindrecord/meta/int | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/meta/int diff --git a/mindspore2022/ccsrc/minddata/mindrecord/meta/int b/mindspore2022/ccsrc/minddata/mindrecord/meta/int deleted file mode 100644 index 87194633898..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/meta/int +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2019-2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "minddata/mindrecord/include/shard_category.h"//按照路径寻找以下文件，导入到本文件 - -namespace mindspore {//创建名为mindspore的空间 -namespace mindrecord {//创建名为mindrecord的空间 -//引用ShardCategory空间中ShardCategory函数,输入三种参数调用指定函数 -//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 -ShardCategory::ShardCategory(const std::vector> &categories, int64_t num_elements, - bool replacement) - : categories_(categories), - category_field_(""), - num_elements_(num_elements), - num_categories_(0), - replacement_(replacement) {} -//引用ShardCategory空间中ShardCategory函数,输入四种参数调用指定函数 -//对参数categories、category_field_、num_elements_、num_categories_、replacement_进行进行参数初始化 -ShardCategory::ShardCategory(const std::string &category_field, int64_t num_elements, int64_t num_categories, - bool replacement) - : categories_({}), - category_field_(category_field), - num_elements_(num_elements), - num_categories_(num_categories), - replacement_(replacement) {} - -Status ShardCategory::Execute(ShardTaskList &tasks) { return Status::OK(); }//在ShardCategory空间中创建Status型Execute函数,返回值为Status空间中的OK函数的返回值 -//在ShardCategory空间中创建int64_t型GetNumSamples函数,返回值为0或-1 -//判断dataset_size的值 -//若dataset_size的值为0,则返回dataset_size本身 -//若dataset_size的值大于0,则继续判断num_classes、num_categories_、num_elements_的值是否大于0。若均大于0,则修改num_classes的赋值,赋值为num_categories_和num_classes的最小值 -//继续判断num_classes是否为0,若为0则返回0。不为0则判断num_elements_是否大于int64_t类型最大值与num_classes的商,若是则返回-1。若均不符合则返回num_classes和num_elements_的乘积 -//若均不符合,则返回0 -int64_t ShardCategory::GetNumSamples(int64_t dataset_size, int64_t num_classes) { - if (dataset_size == 0) return dataset_size; - if (dataset_size > 0 && num_classes > 0 && num_categories_ > 0 && num_elements_ > 0) { - num_classes = std::min(num_categories_, num_classes); - if (num_classes == 0) { - return 0; - } - if (num_elements_ > std::numeric_limits::max() / num_classes) { - return -1; - } - return num_classes * num_elements_; - } - return 0; -} -} // namespace mindrecord -} // namespace mindspore -- 2.34.1 From e35ced83e1e5288b7770b206de0fd0ffd366acca Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:40:24 +0800 Subject: [PATCH 136/160] ADD file via upload --- .../minddata/mindrecord/include/common52Fint | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint b/mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint new file mode 100644 index 00000000000..dfc41aa4a00 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ + +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; +namespace nlohmann { +template <> +struct adl_serializer { + py::object FromJson(const json &j); + + void ToJson(json *j, const py::object &obj); +}; + +namespace detail { +py::object FromJsonImpl(const json &j); + +json ToJsonImpl(const py::handle &obj); +} // namespace detail +} // namespace nlohmann +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -- 2.34.1 From 08ee360477638ab13487265fec18c9f52a3eb59a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:40:38 +0800 Subject: [PATCH 137/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint' --- .../minddata/mindrecord/include/common52Fint | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint b/mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint deleted file mode 100644 index dfc41aa4a00..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/common52Fint +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ - -#include -#include -#include "minddata/mindrecord/include/common/shard_utils.h" -#include "pybind11/pybind11.h" - -namespace py = pybind11; -namespace nlohmann { -template <> -struct adl_serializer { - py::object FromJson(const json &j); - - void ToJson(json *j, const py::object &obj); -}; - -namespace detail { -py::object FromJsonImpl(const json &j); - -json ToJsonImpl(const py::handle &obj); -} // namespace detail -} // namespace nlohmann -#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -- 2.34.1 From bf76d9484734fff72bf36832f40e272b2c85dd4b Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:41:16 +0800 Subject: [PATCH 138/160] ADD file via upload --- .../minddata/mindrecord/include/common/int | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/common/int diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/common/int b/mindspore2022/ccsrc/minddata/mindrecord/include/common/int new file mode 100644 index 00000000000..dfc41aa4a00 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/common/int @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ + +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; +namespace nlohmann { +template <> +struct adl_serializer { + py::object FromJson(const json &j); + + void ToJson(json *j, const py::object &obj); +}; + +namespace detail { +py::object FromJsonImpl(const json &j); + +json ToJsonImpl(const py::handle &obj); +} // namespace detail +} // namespace nlohmann +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -- 2.34.1 From 21d9ba11681e37eca7b8807cc0b930165bb5c2c9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:41:34 +0800 Subject: [PATCH 139/160] ADD file via upload --- .../mindrecord/include/common/shard_pybind.h | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_pybind.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_pybind.h b/mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_pybind.h new file mode 100644 index 00000000000..dfc41aa4a00 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_pybind.h @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ + +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; +namespace nlohmann { +template <> +struct adl_serializer { + py::object FromJson(const json &j); + + void ToJson(json *j, const py::object &obj); +}; + +namespace detail { +py::object FromJsonImpl(const json &j); + +json ToJsonImpl(const py::handle &obj); +} // namespace detail +} // namespace nlohmann +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -- 2.34.1 From 338457b0a25fde99aba0fff89cf755e4af776578 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:41:55 +0800 Subject: [PATCH 140/160] ADD file via upload --- .../mindrecord/include/common/shard_utils.h | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_utils.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_utils.h b/mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_utils.h new file mode 100644 index 00000000000..7f0aaaebb67 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/common/shard_utils.h @@ -0,0 +1,210 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ + +#include +#include +#include +#include +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_error.h" +#include "nlohmann/json.hpp" +#include "./sqlite3.h" +#include "utils/log_adapter.h" + +/* To be used when dlog is ok #include "./slog.h" */ +#ifdef DEBUG +#define MS_ASSERT(f) assert(f) +#else +#define MS_ASSERT(f) ((void)0) +#endif + +namespace mindspore { +namespace mindrecord { +using json = nlohmann::json; + +//定义int型常量KInt1，KInt2，KInt3，kUnsignedInt4 +const int kInt0 = 0; +const int kInt1 = 1; +const int kInt2 = 2; +const int kInt3 = 3; +const int kUnsignedInt4 = 4; + +enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel }; + +const char kVersion[] = "3.0"; //将3.0赋给字符型数组kVersion[] +const std::vector kSupportedVersion = {"2.0", kVersion}; //将2.0，kVersion赋给vector kSupportedVersion + +enum ShardType { + kNLP = 0, + kCV = 1, +}; + +enum TaskType { + kCommonTask = 0, + kPaddedTask = 1, +}; +enum SamplerType { kCustomTopNSampler, kCustomTopPercentSampler, kSubsetRandomSampler, kPKSampler, kSubsetSampler }; + +enum ShuffleType { kShuffleCategory, kShuffleSample }; + +const double kEpsilon = 1e-7; + +const int kThreadNumber = 14; + +// Shard default parameters Shard默认参数 +const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB +const uint64_t kDefaultPageSize = 1 << 25; // 32MB + +// HeaderSize [16KB, 128MB] +const int kMinHeaderSize = 1 << 14; // 16KB +const int kMaxHeaderSize = 1 << 27; // 128MB + +// PageSize [32KB, 256MB] +const int kMinPageSize = 1 << 15; // 32KB +const int kMaxPageSize = 1 << 28; // 256MB + +// used by value length / schema id length / statistic id length ... 由值长度/架构id长度/统计id长度使用。。。 +const uint64_t kInt64Len = 8; + +// Minimum file size 最小文件大小 +const uint64_t kMinFileSize = kInt64Len; + +const int kMinShardCount = 1; +const int kMaxShardCount = 1000; // write +const int kMaxFileCount = 4096; // read + +const int kMinConsumerCount = 1; +const int kMaxConsumerCount = 128; + +const int kMaxSchemaCount = 1; +const int kMaxThreadCount = 32; +const int kMaxFieldCount = 100; + +// Minimum free disk size 最小可用磁盘大小 +const int kMinFreeDiskSize = 10; // 10M + +// dummy json +const json kDummyId = R"({"id": 0})"_json; + +// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) 将模式中的类型转换为sqlite3中的类型（NULL、INTEGER、REAL、TEXT、BLOB） +const std::unordered_map kDbJsonMap = { + {"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"}, + {"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"}, + {"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"}, + {"bytes", "BLOB"}}; + +const char kPoint = '.'; + +const char kPathSeparator = +#if defined(_WIN32) || defined(_WIN64) + '\\'; +#else + '/'; +#endif + +// field type used by check schema validation 检查架构验证使用的字段类型 +const std::set kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"}; + +// can be searched field list 可搜索字段列表 +const std::set kScalarFieldTypeSet = {"string", "int32", "int64", "float32", "float64"}; + +// number field list 数字字段列表 +const std::set kNumberFieldTypeSet = {"int32", "int64", "float32", "float64"}; + +const std::unordered_map kTypesMap = { + {"bool", "int32"}, {"int8", "int32"}, {"uint8", "bytes"}, {"int16", "int32"}, + {"uint16", "int32"}, {"int32", "int32"}, {"uint32", "int64"}, {"int64", "int64"}, + {"float16", "float32"}, {"float32", "float32"}, {"float64", "float64"}, {"string", "string"}}; + +/// \brief the max number of samples to enable lazy load 启用延迟加载的最大样本数 +const uint32_t LAZY_LOAD_THRESHOLD = 5000000; + +/// \brief split a string using a character 使用字符拆分字符串 +/// \param[in] field target string 目标字符串 +/// \param[in] separator a character for splitting 用于拆分的字符 +/// \return vector type result 矢量类型结果 +std::vector StringSplit(const std::string &field, char separator); + +/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' 验证字段名由“0-9”或“a-z”或“a-z”或“_”或“-”组成 +/// \param[in] str target string 目标字符串 +/// \return +bool ValidateFieldName(const std::string &str); + +/// \brief get the filename by the path 通过路径获取文件名 +/// \param s file path 文件路径 +/// \param fn_ptr shared ptr of file name 文件名的共享指针 +/// \return Status +Status GetFileName(const std::string &path, std::shared_ptr *fn_ptr); + +/// \brief get parent dir 获取父目录 +/// \param path file path 文件路径 +/// \param pd_ptr shared ptr of parent path 父路径的共享指针 +/// \return Status +Status GetParentDir(const std::string &path, std::shared_ptr *pd_ptr); + +bool CheckIsValidUtf8(const std::string &str); + +/// \brief judge if a path is legal file 判断路径是否为合法文件 +/// \param path file path 文件路径 +/// \return Whether the path is legal or not 路径是否合法 +bool IsLegalFile(const std::string &path); + +enum DiskSizeType { kTotalSize = 0, kFreeSize }; + +/// \brief get the free space about the disk 获取磁盘的可用空间 +/// \param str_dir file path 文件路径 +/// \param disk_type: kTotalSize / kFreeSize +/// \param size: shared ptr of size in Megabytes 以MB为单位的共享指针 +/// \return Status +Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr *size); + +/// \brief get the max hardware concurrency //获取最大硬件并发 +/// \return max concurrency 最大并发数 +uint32_t GetMaxThreadNum(); + +/// \brief get absolute path of all mindrecord files 获取所有mindrecord文件的绝对路径 +/// \param path path to one fo mindrecord files 一个fo-mindrecord文件的路径 +/// \param addresses relative path of all mindrecord files 所有思维记录文件的相对路径 +/// \param ds shared ptr of vector of absolute path 绝对路径矢量的共享指针 +/// \return Status +Status GetDatasetFiles(const std::string &path, const json &addresses, std::shared_ptr> *ds); +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -- 2.34.1 From 0529206ec0594221ffc7c479aee0b96d8340a36e Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:42:14 +0800 Subject: [PATCH 141/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/common/int' --- .../minddata/mindrecord/include/common/int | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/common/int diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/common/int b/mindspore2022/ccsrc/minddata/mindrecord/include/common/int deleted file mode 100644 index dfc41aa4a00..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/common/int +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ - -#include -#include -#include "minddata/mindrecord/include/common/shard_utils.h" -#include "pybind11/pybind11.h" - -namespace py = pybind11; -namespace nlohmann { -template <> -struct adl_serializer { - py::object FromJson(const json &j); - - void ToJson(json *j, const py::object &obj); -}; - -namespace detail { -py::object FromJsonImpl(const json &j); - -json ToJsonImpl(const py::handle &obj); -} // namespace detail -} // namespace nlohmann -#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_PYBIND_H_ -- 2.34.1 From a1696007ca0e7728afd33274d40771a151b1ed82 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:42:50 +0800 Subject: [PATCH 142/160] Delete 'mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h' --- .../minddata/mindrecord/include/shard_utils.h | 210 ------------------ 1 file changed, 210 deletions(-) delete mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h deleted file mode 100644 index 056d086da08..00000000000 --- a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_utils.h +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ - -#include -#include -#include -#include -#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "minddata/mindrecord/include/shard_error.h" -#include "nlohmann/json.hpp" -#include "./sqlite3.h" -#include "utils/log_adapter.h" - -/* To be used when dlog is ok #include "./slog.h" */ -#ifdef DEBUG -#define MS_ASSERT(f) assert(f) -#else -#define MS_ASSERT(f) ((void)0) -#endif - -namespace mindspore {// -namespace mindrecord { -using json = nlohmann::json; - -//定义int型常量KInt1，KInt2，KInt3，kUnsignedInt4 -const int kInt0 = 0; -const int kInt1 = 1; -const int kInt2 = 2; -const int kInt3 = 3; -const int kUnsignedInt4 = 4; - -enum LabelCategory { kSchemaLabel, kStatisticsLabel, kIndexLabel }; - -const char kVersion[] = "3.0"; //将3.0赋给字符型数组kVersion[] -const std::vector kSupportedVersion = {"2.0", kVersion}; //将2.0，kVersion赋给vector kSupportedVersion - -enum ShardType { - kNLP = 0, - kCV = 1, -}; - -enum TaskType { - kCommonTask = 0, - kPaddedTask = 1, -}; -enum SamplerType { kCustomTopNSampler, kCustomTopPercentSampler, kSubsetRandomSampler, kPKSampler, kSubsetSampler }; - -enum ShuffleType { kShuffleCategory, kShuffleSample }; - -const double kEpsilon = 1e-7; - -const int kThreadNumber = 14; - -// Shard default parameters Shard默认参数 -const uint64_t kDefaultHeaderSize = 1 << 24; // 16MB -const uint64_t kDefaultPageSize = 1 << 25; // 32MB - -// HeaderSize [16KB, 128MB] -const int kMinHeaderSize = 1 << 14; // 16KB -const int kMaxHeaderSize = 1 << 27; // 128MB - -// PageSize [32KB, 256MB] -const int kMinPageSize = 1 << 15; // 32KB -const int kMaxPageSize = 1 << 28; // 256MB - -// used by value length / schema id length / statistic id length ... 由值长度/架构id长度/统计id长度使用。。。 -const uint64_t kInt64Len = 8; - -// Minimum file size 最小文件大小 -const uint64_t kMinFileSize = kInt64Len; - -const int kMinShardCount = 1; -const int kMaxShardCount = 1000; // write -const int kMaxFileCount = 4096; // read - -const int kMinConsumerCount = 1; -const int kMaxConsumerCount = 128; - -const int kMaxSchemaCount = 1; -const int kMaxThreadCount = 32; -const int kMaxFieldCount = 100; - -// Minimum free disk size 最小可用磁盘大小 -const int kMinFreeDiskSize = 10; // 10M - -// dummy json -const json kDummyId = R"({"id": 0})"_json; - -// translate type in schema to type in sqlite3(NULL, INTEGER, REAL, TEXT, BLOB) 将模式中的类型转换为sqlite3中的类型（NULL、INTEGER、REAL、TEXT、BLOB） -const std::unordered_map kDbJsonMap = { - {"string", "TEXT"}, {"date", "DATE"}, {"date-time", "DATETIME"}, {"null", "NULL"}, - {"integer", "INTEGER"}, {"boolean", "BOOLEAN"}, {"array", "BLOB"}, {"number", "NUMERIC"}, - {"int32", "INTEGER"}, {"int64", "INTEGER"}, {"float32", "NUMERIC"}, {"float64", "NUMERIC"}, - {"bytes", "BLOB"}}; - -const char kPoint = '.'; - -const char kPathSeparator = -#if defined(_WIN32) || defined(_WIN64) - '\\'; -#else - '/'; -#endif - -// field type used by check schema validation 检查架构验证使用的字段类型 -const std::set kFieldTypeSet = {"bytes", "string", "int32", "int64", "float32", "float64"}; - -// can be searched field list 可搜索字段列表 -const std::set kScalarFieldTypeSet = {"string", "int32", "int64", "float32", "float64"}; - -// number field list 数字字段列表 -const std::set kNumberFieldTypeSet = {"int32", "int64", "float32", "float64"}; - -const std::unordered_map kTypesMap = { - {"bool", "int32"}, {"int8", "int32"}, {"uint8", "bytes"}, {"int16", "int32"}, - {"uint16", "int32"}, {"int32", "int32"}, {"uint32", "int64"}, {"int64", "int64"}, - {"float16", "float32"}, {"float32", "float32"}, {"float64", "float64"}, {"string", "string"}}; - -/// \brief the max number of samples to enable lazy load 启用延迟加载的最大样本数 -const uint32_t LAZY_LOAD_THRESHOLD = 5000000; - -/// \brief split a string using a character 使用字符拆分字符串 -/// \param[in] field target string 目标字符串 -/// \param[in] separator a character for splitting 用于拆分的字符 -/// \return vector type result 矢量类型结果 -std::vector StringSplit(const std::string &field, char separator); - -/// \brief validate field name is composed of '0-9' or 'a-z' or 'A-Z' or '_' or '-' 验证字段名由“0-9”或“a-z”或“a-z”或“_”或“-”组成 -/// \param[in] str target string 目标字符串 -/// \return -bool ValidateFieldName(const std::string &str); - -/// \brief get the filename by the path 通过路径获取文件名 -/// \param s file path 文件路径 -/// \param fn_ptr shared ptr of file name 文件名的共享指针 -/// \return Status -Status GetFileName(const std::string &path, std::shared_ptr *fn_ptr); - -/// \brief get parent dir 获取父目录 -/// \param path file path 文件路径 -/// \param pd_ptr shared ptr of parent path 父路径的共享指针 -/// \return Status -Status GetParentDir(const std::string &path, std::shared_ptr *pd_ptr); - -bool CheckIsValidUtf8(const std::string &str); - -/// \brief judge if a path is legal file 判断路径是否为合法文件 -/// \param path file path 文件路径 -/// \return Whether the path is legal or not 路径是否合法 -bool IsLegalFile(const std::string &path); - -enum DiskSizeType { kTotalSize = 0, kFreeSize }; - -/// \brief get the free space about the disk 获取磁盘的可用空间 -/// \param str_dir file path 文件路径 -/// \param disk_type: kTotalSize / kFreeSize -/// \param size: shared ptr of size in Megabytes 以MB为单位的共享指针 -/// \return Status -Status GetDiskSize(const std::string &str_dir, const DiskSizeType &disk_type, std::shared_ptr *size); - -/// \brief get the max hardware concurrency //获取最大硬件并发 -/// \return max concurrency 最大并发数 -uint32_t GetMaxThreadNum(); - -/// \brief get absolute path of all mindrecord files 获取所有mindrecord文件的绝对路径 -/// \param path path to one fo mindrecord files 一个fo-mindrecord文件的路径 -/// \param addresses relative path of all mindrecord files 所有思维记录文件的相对路径 -/// \param ds shared ptr of vector of absolute path 绝对路径矢量的共享指针 -/// \return Status -Status GetDatasetFiles(const std::string &path, const json &addresses, std::shared_ptr> *ds); -} // namespace mindrecord -} // namespace mindspore - -#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_COMMON_SHARD_UTILS_H_ -- 2.34.1 From bcd9fb376d5003cdce23274e4c6324394bce6566 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:43:21 +0800 Subject: [PATCH 143/160] ADD file via upload --- .../mindrecord/include/shard_column.h | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_column.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_column.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_column.h new file mode 100644 index 00000000000..f2978dbcb40 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_column.h @@ -0,0 +1,175 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_COLUMN_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_COLUMN_H_ + +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_header.h" + +namespace mindspore { +namespace mindrecord { +const uint64_t kUnsignedOne = 1; +const uint64_t kBitsOfByte = 8; +const uint64_t kDataTypeBits = 2; +const uint64_t kNumDataOfByte = 4; +const uint64_t kBytesOfColumnLen = 4; +const uint64_t kDataTypeBitMask = 3; +const uint64_t kDataTypes = 6; + +enum IntegerType { kInt8Type = 0, kInt16Type, kInt32Type, kInt64Type }; + +enum ColumnCategory { ColumnInRaw, ColumnInBlob, ColumnNotFound }; + +enum ColumnDataType { + ColumnBytes = 0, + ColumnString = 1, + ColumnInt32 = 2, + ColumnInt64 = 3, + ColumnFloat32 = 4, + ColumnFloat64 = 5, + ColumnNoDataType = 6 +}; + +const uint32_t ColumnDataTypeSize[kDataTypes] = {1, 1, 4, 8, 4, 8}; + +const std::vector ColumnDataTypeNameNormalized = {"uint8", "string", "int32", + "int64", "float32", "float64"}; + +const std::unordered_map ColumnDataTypeMap = { + {"bytes", ColumnBytes}, {"string", ColumnString}, {"int32", ColumnInt32}, + {"int64", ColumnInt64}, {"float32", ColumnFloat32}, {"float64", ColumnFloat64}}; + +class __attribute__((visibility("default"))) ShardColumn { + public: + explicit ShardColumn(const std::shared_ptr &shard_header, bool compress_integer = true); + explicit ShardColumn(const json &schema_json, bool compress_integer = true); + + ~ShardColumn() = default; + + /// \brief get column value by column name + Status GetColumnValueByName(const std::string &column_name, const std::vector &columns_blob, + const json &columns_json, const unsigned char **data, + std::unique_ptr *data_ptr, uint64_t *const n_bytes, + ColumnDataType *column_data_type, uint64_t *column_data_type_size, + std::vector *column_shape); + + /// \brief compress blob + std::vector CompressBlob(const std::vector &blob, int64_t *compression_size); + + /// \brief check if blob compressed + bool CheckCompressBlob() const { return has_compress_blob_; } + + /// \brief getter + uint64_t GetNumBlobColumn() const { return num_blob_column_; } + + /// \brief getter + std::vector GetColumnName() { return column_name_; } + + /// \brief getter + std::vector GeColumnDataType() { return column_data_type_; } + + /// \brief getter + std::vector> GetColumnShape() { return column_shape_; } + + /// \brief get column value from blob + Status GetColumnFromBlob(const std::string &column_name, const std::vector &columns_blob, + const unsigned char **data, std::unique_ptr *data_ptr, + uint64_t *const n_bytes); + + /// \brief get column type + Status GetColumnTypeByName(const std::string &column_name, ColumnDataType *column_data_type, + uint64_t *column_data_type_size, std::vector *column_shape, + ColumnCategory *column_category); + + /// \brief get column value from json + Status GetColumnFromJson(const std::string &column_name, const json &columns_json, + std::unique_ptr *data_ptr, uint64_t *n_bytes); + + private: + /// \brief initialization + void Init(const json &schema_json, bool compress_integer = true); + + /// \brief get float value from json + template + Status GetFloat(std::unique_ptr *data_ptr, const json &json_column_value, bool use_double); + + /// \brief get integer value from json + template + Status GetInt(std::unique_ptr *data_ptr, const json &json_column_value); + + /// \brief get column offset address and size from blob + Status GetColumnAddressInBlock(const uint64_t &column_id, const std::vector &columns_blob, + uint64_t *num_bytes, uint64_t *shift_idx); + + /// \brief check if column name is available + ColumnCategory CheckColumnName(const std::string &column_name); + + /// \brief compress integer column + static vector CompressInt(const vector &src_bytes, const IntegerType &int_type); + + /// \brief uncompress integer array column + template + static Status UncompressInt(const uint64_t &column_id, std::unique_ptr *const data_ptr, + const std::vector &columns_blob, uint64_t *num_bytes, uint64_t shift_idx); + + /// \brief convert big-endian bytes to unsigned int + /// \param bytes_array bytes array + /// \param pos shift address in bytes array + /// \param i_type integer type + /// \return unsigned int + static uint64_t BytesBigToUInt64(const std::vector &bytes_array, const uint64_t &pos, + const IntegerType &i_type); + + /// \brief convert unsigned int to big-endian bytes + /// \param value integer value + /// \param i_type integer type + /// \return bytes + static std::vector UIntToBytesBig(uint64_t value, const IntegerType &i_type); + + /// \brief convert unsigned int to little-endian bytes + /// \param value integer value + /// \param i_type integer type + /// \return bytes + static std::vector UIntToBytesLittle(uint64_t value, const IntegerType &i_type); + + /// \brief convert unsigned int to little-endian bytes + /// \param bytes_array bytes array + /// \param pos shift address in bytes array + /// \param src_i_type source integer typ0e + /// \param dst_i_type (output), destination integer type + /// \return integer + static int64_t BytesLittleToMinIntType(const std::vector &bytes_array, const uint64_t &pos, + const IntegerType &src_i_type, IntegerType *dst_i_type = nullptr); + + private: + std::vector column_name_; // column name list + std::vector column_data_type_; // column data type list + std::vector> column_shape_; // column shape list + std::unordered_map column_name_id_; // column name id map + std::vector blob_column_; // blob column list + std::unordered_map blob_column_id_; // blob column name id map + bool has_compress_blob_; // if has compress blob + uint64_t num_blob_column_; // number of blob columns +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_COLUMN_H_ -- 2.34.1 From 79a889bab9fbe88a2d27e4d93165c0e019f98923 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:43:37 +0800 Subject: [PATCH 144/160] ADD file via upload --- .../include/shard_distributed_sample.h | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_distributed_sample.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_distributed_sample.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_distributed_sample.h new file mode 100644 index 00000000000..3bf488a0963 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_distributed_sample.h @@ -0,0 +1,55 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_DISTRIBUTED_SAMPLE_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_DISTRIBUTED_SAMPLE_H_ + +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_operator.h" +#include "minddata/mindrecord/include/shard_shuffle.h" +#include "minddata/mindrecord/include/shard_sample.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardDistributedSample : public ShardSample { + public: + ShardDistributedSample(int num_shards, int shard_id, int64_t no_of_padded_samples, bool shuffle, uint32_t seed, + int64_t no_of_samples = 0, int64_t offset = -1); + + ShardDistributedSample(int num_shards, int shard_id, bool shuffle, uint32_t seed, int64_t no_of_samples = 0, + int64_t offset = -1); + + void SetNumPaddedSamples(int64_t no_of_padded_samples) { no_of_padded_samples_ = no_of_padded_samples; } + + ~ShardDistributedSample() override{}; + + Status PreExecute(ShardTaskList &tasks) override; + + int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override; + + private: + bool shuffle_; + int64_t no_of_padded_samples_; + bool first_epoch_; // check (num_sample + num_padded) % num_shards == 0 in first epoch + ShardTaskList task_; // maintain the input tasks in first epoch +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_DISTRIBUTED_SAMPLE_H_ -- 2.34.1 From 5358d708a1df0c440645756b6415833c7b107279 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:43:56 +0800 Subject: [PATCH 145/160] ADD file via upload --- .../minddata/mindrecord/include/shard_error.h | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_error.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_error.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_error.h new file mode 100644 index 00000000000..9f76d9b8f24 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_error.h @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_ERROR_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_ERROR_H_ + +#include +#include +#include "include/api/status.h" + +namespace mindspore { //命名mindspore空间 +namespace mindrecord { //命名mindrecord空间 +#define RETURN_IF_NOT_OK(_s) \ //参数_s + do { \ + Status __rc = (_s); \ //将_s赋给_rc + if (__rc.IsError()) { \ //如果_rc.IsError()返回值为1，则返回_rc + return __rc; \ + } \ + } while (false) //停止循环 + +#define RELEASE_AND_RETURN_IF_NOT_OK(_s, _db, _in) \ //参数_s,_db,_in + do { \ + Status __rc = (_s); \ //将_s赋给_rc + if (__rc.IsError()) { \ //当_rc.IsError()返回值为1时，如果_db不为空，关闭_db数据库链接 + if ((_db) != nullptr) { \ + sqlite3_close(_db); \ + } \ + (_in).close(); \ //关闭_in + return __rc; \ //返回_rc + } \ + } while (false) //停止循环 + +#define CHECK_FAIL_RETURN_UNEXPECTED(_condition, _e) \ //参数_condition,_e + do { \ + if (!(_condition)) { \ //如果_condition为0，返回StatusCOde下的kMDUnexpectedError, __LINE__, __FILE__, _e + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, _e); \ + } \ + } while (false) //停止循环 + +#define RETURN_UNEXPECTED_IF_NULL(_ptr) \ //参数_ptr + do { \ + if ((_ptr) == nullptr) { \ //如果_ptr为空指针，则err_msg为The pointer[" + std::string(#_ptr) + "] is null + std::string err_msg = "The pointer[" + std::string(#_ptr) + "] is null."; \ + RETURN_STATUS_UNEXPECTED(err_msg); \ //将err_msg传入RETURN_STATUS_UNEXPECTED()函数，返回StatusCOde下的kMDUnexpectedError, __LINE__, __FILE__,err_msg + } \ + } while (false) //停止循环 + +#define RETURN_STATUS_UNEXPECTED(_e) \ //参数_e + do { \ + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, _e); \ //返回StatusCOde下的kMDUnexpectedError, __LINE__, __FILE__, _e + } while (false) //停止循环 + +enum MSRStatus { //失败 + SUCCESS = 0, + FAILED = 1, +}; + +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_ERROR_H_ -- 2.34.1 From b08ea1f3c34d2740330f9328a3b189add0182d6b Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:44:11 +0800 Subject: [PATCH 146/160] ADD file via upload --- .../mindrecord/include/shard_header.h | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_header.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_header.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_header.h new file mode 100644 index 00000000000..b0d1a5e38d4 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_header.h @@ -0,0 +1,200 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_HEADER_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_HEADER_H_ + +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "minddata/mindrecord/include/shard_index.h" +#include "minddata/mindrecord/include/shard_page.h" +#include "minddata/mindrecord/include/shard_schema.h" +#include "minddata/mindrecord/include/shard_statistics.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardHeader { + public: + ShardHeader(); + + ~ShardHeader() = default; + + Status BuildDataset(const std::vector &file_paths, bool load_dataset = true); + + static Status BuildSingleHeader(const std::string &file_path, std::shared_ptr *header_ptr); + /// \brief add the schema and save it 添加架构并保存 + /// \param[in] schema the schema needs to be added 需要添加的架构 + /// \return the last schema's id 返回最后一个架构的 ID + int AddSchema(std::shared_ptr schema); + + /// \brief add the statistic and save it 添加统计信息并保存 + /// \param[in] statistic the statistic needs to be added 需要添加的统计数据 + /// \return the last statistic's id 返回最后一个统计信息的 ID + void AddStatistic(std::shared_ptr statistic); + + /// \brief create index and add fields which from schema for each schema 创建索引并添加来自每个架构的架构的字段 + /// \param[in] fields the index fields needs to be added 需要添加的索引字段 + /// \return SUCCESS if add successfully, FAILED if not 如果添加成功，返回SUCCESS，否则返回FAILED + Status AddIndexFields(std::vector> fields); + + Status AddIndexFields(const std::vector &fields); + + /// \brief get the schema 获取架构 + /// \return the schema 返回架构 + std::vector> GetSchemas(); + + /// \brief get Statistics 获取统计数据 + /// \return the Statistic 返回统计数据 + std::vector> GetStatistics(); + + /// \brief add the statistic and save it 添加统计信息并保存 + /// \param[in] statistic info of slim size 苗条尺寸的统计信息 + /// \return null 返回空 + int64_t GetSlimSizeStatistic(const json &slim_size_json); + + /// \brief get the fields of the index 获取索引的字段 + /// \return the fields of the index 返回索引的字段 + std::vector> GetFields(); + + /// \brief get the index + /// \return the index + std::shared_ptr GetIndex(); + + /// \brief get the schema by schemaid + /// \param[in] schema_id the id of schema needs to be got + /// \param[in] schema_ptr the schema obtained by schemaId + /// \return Status + Status GetSchemaByID(int64_t schema_id, std::shared_ptr *schema_ptr); + + /// \brief get the filepath to shard by shardID + /// \param[in] shardID the id of shard which filepath needs to be obtained + /// \return the filepath obtained by shardID + std::string GetShardAddressByID(int64_t shard_id); + + /// \brief get the statistic by statistic id + /// \param[in] statistic_id the id of statistic needs to be get + /// \param[in] statistics_ptr the statistics obtained by statistic id + /// \return Status + Status GetStatisticByID(int64_t statistic_id, std::shared_ptr *statistics_ptr); + + Status InitByFiles(const std::vector &file_paths); + + void SetIndex(Index index) { index_ = std::make_shared(index); } + + Status GetPage(const int &shard_id, const int &page_id, std::shared_ptr *page_ptr); + + Status SetPage(const std::shared_ptr &new_page); + + Status AddPage(const std::shared_ptr &new_page); + + int64_t GetLastPageId(const int &shard_id); + + int GetLastPageIdByType(const int &shard_id, const std::string &page_type); + + Status GetPageByGroupId(const int &group_id, const int &shard_id, std::shared_ptr *page_ptr); + + std::vector GetShardAddresses() const { return shard_addresses_; } + + int GetShardCount() const { return shard_count_; } + + int GetSchemaCount() const { return schema_.size(); } + + uint64_t GetHeaderSize() const { return header_size_; } + + uint64_t GetPageSize() const { return page_size_; } + + uint64_t GetCompressionSize() const { return compression_size_; } + + void SetHeaderSize(const uint64_t &header_size) { header_size_ = header_size; } + + void SetPageSize(const uint64_t &page_size) { page_size_ = page_size; } + + void SetCompressionSize(const uint64_t &compression_size) { compression_size_ = compression_size; } + + std::vector SerializeHeader(); + + Status PagesToFile(const std::string dump_file_name); + + Status FileToPages(const std::string dump_file_name); + + static Status Initialize(const std::shared_ptr *header_ptr, const json &schema, + const std::vector &index_fields, std::vector &blob_fields, + uint64_t &schema_id); + + private: + Status InitializeHeader(const std::vector &headers, bool load_dataset); + + /// \brief get the headers from all the shard data + /// \param[in] the shard data real path + /// \param[in] the headers which read from the shard data + /// \return SUCCESS/FAILED + Status GetHeaders(const vector &real_addresses, std::vector &headers); + + Status ValidateField(const std::vector &field_name, json schema, const uint64_t &schema_id); + + /// \brief check the binary file status + static Status CheckFileStatus(const std::string &path); + + static Status ValidateHeader(const std::string &path, std::shared_ptr *header_ptr); + + void GetHeadersOneTask(int start, int end, std::vector &headers, const vector &realAddresses); + + Status ParseIndexFields(const json &index_fields); + + Status CheckIndexField(const std::string &field, const json &schema); + + Status ParsePage(const json &page, int shard_index, bool load_dataset); + + Status ParseStatistics(const json &statistics); + + Status ParseSchema(const json &schema); + + void ParseShardAddress(const json &address); + + std::string SerializeIndexFields(); + + std::vector SerializePage(); + + std::string SerializeStatistics(); + + std::string SerializeSchema(); + + std::string SerializeShardAddress(); + + std::shared_ptr InitIndexPtr(); + + Status GetAllSchemaID(std::set &bucket_count); + + uint32_t shard_count_; + uint64_t header_size_; + uint64_t page_size_; + uint64_t compression_size_; + + std::shared_ptr index_; + std::vector shard_addresses_; + std::vector> schema_; + std::vector> statistics_; + std::vector>> pages_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_HEADER_H_ -- 2.34.1 From 4c7cab599299f1f2ccdf10b34cb71758541e4cc4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:44:25 +0800 Subject: [PATCH 147/160] ADD file via upload --- .../minddata/mindrecord/include/shard_index.h | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_index.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_index.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_index.h new file mode 100644 index 00000000000..cdc0d8f3f0d --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_index.h @@ -0,0 +1,65 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INDEX_H +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INDEX_H +#pragma once + +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "minddata/mindrecord/include/shard_schema.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +using std::cin; +using std::endl; +using std::pair; +using std::string; +using std::vector; + +class __attribute__((visibility("default"))) Index { + public: + Index(); + + ~Index() {} + + /// \brief Add field which from schema according to schemaId + /// \param[in] schemaId the id of schema to be added + /// \param[in] field the field need to be added + /// + /// add the field to the fields_ vector + void AddIndexField(const int64_t &schemaId, const std::string &field); + + /// \brief get stored fields + /// \return fields stored + std::vector > GetFields(); + + private: + std::vector > fields_; + string database_name_; + string table_name_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INDEX_H -- 2.34.1 From 61d4e6279ca9364222c98b8b645be2e9869c3bbb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:44:40 +0800 Subject: [PATCH 148/160] ADD file via upload --- .../include/shard_index_generator.h | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_index_generator.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_index_generator.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_index_generator.h new file mode 100644 index 00000000000..a2a29c619cf --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_index_generator.h @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_header.h" +#include "./sqlite3.h" + +namespace mindspore { +namespace mindrecord { +using INDEX_FIELDS = std::vector>; +using ROW_DATA = std::vector>>; +class __attribute__((visibility("default"))) ShardIndexGenerator { + public: + explicit ShardIndexGenerator(const std::string &file_path, bool append = false); + + Status Build(); + + static Status GenerateFieldName(const std::pair &field, std::shared_ptr *fn_ptr); + + ~ShardIndexGenerator() {} + + /// \brief fetch value in json by field name + /// \param[in] field + /// \param[in] input + /// \param[in] value + /// \return Status + Status GetValueByField(const string &field, const json &input, std::shared_ptr *value); + + /// \brief fetch field type in schema n by field path + /// \param[in] field_path + /// \param[in] schema + /// \return the type of field + static std::string TakeFieldType(const std::string &field_path, json &schema); + + /// \brief create databases for indexes + Status WriteToDatabase(); + + static Status Finalize(const std::vector file_names); + + private: + static int Callback(void *not_used, int argc, char **argv, char **az_col_name); + + static Status ExecuteSQL(const std::string &statement, sqlite3 *db, const string &success_msg = ""); + + static std::string ConvertJsonToSQL(const std::string &json); + + Status CreateDatabase(int shard_no, sqlite3 **db); + + Status GetSchemaDetails(const std::vector &schema_lens, std::fstream &in, + std::shared_ptr> *detail_ptr); + + static Status GenerateRawSQL(const std::vector> &fields, + std::shared_ptr *sql_ptr); + + Status CheckDatabase(const std::string &shard_address, sqlite3 **db); + + /// + /// \param shard_no + /// \param blob_id_to_page_id + /// \param raw_page_id + /// \param in + /// \return Status + Status GenerateRowData(int shard_no, const std::map &blob_id_to_page_id, int raw_page_id, std::fstream &in, + std::shared_ptr *row_data_ptr); + /// + /// \param db + /// \param sql + /// \param data + /// \return + Status BindParameterExecuteSQL(sqlite3 *db, const std::string &sql, const ROW_DATA &data); + + Status GenerateIndexFields(const std::vector &schema_detail, std::shared_ptr *index_fields_ptr); + + Status ExecuteTransaction(const int &shard_no, sqlite3 *db, const std::vector &raw_page_ids, + const std::map &blob_id_to_page_id); + + Status CreateShardNameTable(sqlite3 *db, const std::string &shard_name); + + Status AddBlobPageInfo(std::vector> &row_data, + const std::shared_ptr cur_blob_page, uint64_t &cur_blob_page_offset, std::fstream &in); + + Status AddIndexFieldByRawData(const std::vector &schema_detail, + std::vector> &row_data); + + void DatabaseWriter(); // worker thread + + std::string file_path_; + bool append_; + ShardHeader shard_header_; + uint64_t page_size_; + uint64_t header_size_; + int schema_count_; + std::atomic_int task_; + std::atomic_bool write_success_; + std::vector> fields_; +}; +} // namespace mindrecord +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_INDEX_GENERATOR_H_ -- 2.34.1 From 2bc76fc31990ca3bc0b1b71510e6c3cfaa0aa256 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:44:57 +0800 Subject: [PATCH 149/160] ADD file via upload --- .../mindrecord/include/shard_operator.h | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_operator.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_operator.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_operator.h new file mode 100644 index 00000000000..82217ffb6de --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_operator.h @@ -0,0 +1,85 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_OPERATOR_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_OPERATOR_H_ + +#include +#include +#include "minddata/mindrecord/include/shard_task_list.h" +#include "minddata/dataset/include/dataset/constants.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardOperator { + public: + virtual ~ShardOperator() = default; + + Status operator()(ShardTaskList &tasks) { + RETURN_IF_NOT_OK(this->PreExecute(tasks)); + RETURN_IF_NOT_OK(this->Execute(tasks)); + RETURN_IF_NOT_OK(this->SufExecute(tasks)); + return Status::OK(); + } + + virtual bool HasChildOp() { return child_op_ != nullptr; } + + virtual Status SetChildOp(const std::shared_ptr &child_op) { + if (child_op != nullptr) { + child_op_ = child_op; + } + return Status::OK(); + } + + virtual std::shared_ptr GetChildOp() { return child_op_; } + + virtual Status PreExecute(ShardTaskList &tasks) { return Status::OK(); } + + virtual Status Execute(ShardTaskList &tasks) = 0; + + virtual Status SufExecute(ShardTaskList &tasks) { return Status::OK(); } + + /// \brief compute actual the num_samples via loading data + virtual int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) { return 0; } + + /// \brief Getter the number of samples which is set via python api + virtual int64_t GetNumSamples() const { return num_samples_; } + + /// \brief Setter the number of samples in python + virtual void SetNumSamples(int64_t num_samples) { num_samples_ = num_samples; } + + virtual void UpdateShuffleMode(dataset::ShuffleMode shuffle_mode) { shuffle_mode_ = shuffle_mode; } + + virtual dataset::ShuffleMode GetShuffleMode() { return shuffle_mode_; } + + virtual void SetShardSampleCount(const std::vector &shard_sample_count) { + shard_sample_count_ = shard_sample_count; + } + + virtual std::vector GetShardSampleCount() { return shard_sample_count_; } + + private: + int64_t num_samples_ = 0; + std::shared_ptr child_op_ = nullptr; + // indicate shard_id : inc_count + // 0 : 15 - shard0 has 15 samples + // 1 : 41 - shard1 has 26 samples + // 2 : 58 - shard2 has 17 samples + std::vector shard_sample_count_; + dataset::ShuffleMode shuffle_mode_ = dataset::ShuffleMode::kGlobal; +}; +} // namespace mindrecord +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_OPERATOR_H_ -- 2.34.1 From 70a151d8395db550953712379e69d9c838b98f5e Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:45:15 +0800 Subject: [PATCH 150/160] ADD file via upload --- .../minddata/mindrecord/include/shard_page.h | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_page.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_page.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_page.h new file mode 100644 index 00000000000..2f4c1187f64 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_page.h @@ -0,0 +1,106 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PAGE_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PAGE_H_ + +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "pybind11/pybind11.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +const std::string kPageTypeRaw = "RAW_DATA"; +const std::string kPageTypeBlob = "BLOB_DATA"; +const std::string kPageTypeNewColumn = "NEW_COLUMN_DATA"; + +class __attribute__((visibility("default"))) Page { + public: + Page(const int &page_id, const int &shard_id, const std::string &page_type, const int &page_type_id, + const uint64_t &start_row_id, const uint64_t end_row_id, + const std::vector> &row_group_ids, const uint64_t page_size) + : page_id_(page_id), + shard_id_(shard_id), + page_type_(page_type), + page_type_id_(page_type_id), + start_row_id_(start_row_id), + end_row_id_(end_row_id), + row_group_ids_(row_group_ids), + page_size_(page_size) {} + + ~Page() = default; + + /// \brief get the page and its description + /// \return the json format of the page and its description + json GetPage() const; + + int GetPageID() const { return page_id_; } + + int GetShardID() const { return shard_id_; } + + int GetPageTypeID() const { return page_type_id_; } + + std::string GetPageType() const { return page_type_; } + + uint64_t GetPageSize() const { return page_size_; } + + uint64_t GetStartRowID() const { return start_row_id_; } + + uint64_t GetEndRowID() const { return end_row_id_; } + + void SetEndRowID(const uint64_t &end_row_id) { end_row_id_ = end_row_id; } + + void SetPageSize(const uint64_t &page_size) { page_size_ = page_size; } + + std::pair GetLastRowGroupID() const { return row_group_ids_.back(); } + + std::vector> GetRowGroupIds() const { return row_group_ids_; } + + void SetRowGroupIds(const std::vector> &last_row_group_ids) { + row_group_ids_ = last_row_group_ids; + } + + void DeleteLastGroupId(); + + private: + int page_id_; + int shard_id_; + std::string page_type_; + int page_type_id_; + uint64_t start_row_id_; + uint64_t end_row_id_; + std::vector> row_group_ids_; + uint64_t page_size_; + // JSON page: { + // "page_id":X, + // "shard_id":X, + // "page_type":"XXX", (enum "raw_data", "blob_data", "new_column") + // "page_type_id":X, + // "start_row_id":X, + // "end_row_id":X, + // "row_group_ids":[{"id":X, "offset":X}], + // "page_size":X, +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PAGE_H_ -- 2.34.1 From e8b017c83e756ac4b01f9215b172a3f9728d8625 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:45:30 +0800 Subject: [PATCH 151/160] ADD file via upload --- .../mindrecord/include/shard_pk_sample.h | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_pk_sample.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_pk_sample.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_pk_sample.h new file mode 100644 index 00000000000..090d36a1f0b --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_pk_sample.h @@ -0,0 +1,53 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PK_SAMPLE_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PK_SAMPLE_H_ + +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_operator.h" +#include "minddata/mindrecord/include/shard_shuffle.h" +#include "minddata/mindrecord/include/shard_category.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardPkSample : public ShardCategory { + public: + ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples); + + ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, int64_t num_samples); + + ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_categories, uint32_t seed, + int64_t num_samples); + + ~ShardPkSample() override{}; + + Status SufExecute(ShardTaskList &tasks) override; + + int64_t GetNumSamples() const { return num_samples_; } + + private: + bool shuffle_; + std::shared_ptr shuffle_op_; + int64_t num_samples_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_PK_SAMPLE_H_ -- 2.34.1 From 0a631fd3d7d8d5f4e863695c589141610deb89c9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:45:45 +0800 Subject: [PATCH 152/160] ADD file via upload --- .../mindrecord/include/shard_reader.h | 361 ++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_reader.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_reader.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_reader.h new file mode 100644 index 00000000000..c54e6e5e362 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_reader.h @@ -0,0 +1,361 @@ +/** + * Copyright 2019-2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_READER_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_READER_H_ + +#include +#include +#if !defined(_WIN32) && !defined(_WIN64) && !defined(__APPLE__) +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_category.h" +#include "minddata/mindrecord/include/shard_column.h" +#include "minddata/mindrecord/include/shard_distributed_sample.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "minddata/mindrecord/include/shard_index_generator.h" +#include "minddata/mindrecord/include/shard_operator.h" +#include "minddata/mindrecord/include/shard_pk_sample.h" +#include "minddata/mindrecord/include/shard_reader.h" +#include "minddata/mindrecord/include/shard_sample.h" +#include "minddata/mindrecord/include/shard_shuffle.h" +#include "utils/log_adapter.h" + +#define API_PUBLIC __attribute__((visibility("default"))) + +namespace mindspore { +namespace mindrecord { +using ROW_GROUPS = std::pair>>, std::vector>>; +using ROW_GROUP_BRIEF = std::tuple>, std::vector>; +using TASK_CONTENT = std::pair, json>>>; +const int kNumBatchInMap = 1000; // iterator buffer size in row-reader mode + +class API_PUBLIC ShardReader { + public: + ShardReader(); + + virtual ~ShardReader(); + + /// \brief open files and initialize reader, c++ API + /// \param[in] file_paths the path of ONE file, any file in dataset is fine or file list + /// \param[in] load_dataset load dataset from single file or not + /// \param[in] n_consumer number of threads when reading + /// \param[in] selected_columns column list to be populated + /// \param[in] operators operators applied to data, operator type is shuffle, sample or category + /// \param[in] num_padded the number of padded samples + /// \param[in] lazy_load if the mindrecord dataset is too large, enable lazy load mode to speed up initialization + /// \return MSRStatus the status of MSRStatus + Status Open(const std::vector &file_paths, bool load_dataset, int n_consumer = 4, + const std::vector &selected_columns = {}, + const std::vector> &operators = {}, const int64_t num_padded = 0, + bool lazy_load = false); + + /// \brief close reader + /// \return null + void Close(); + + /// \brief read the file, get schema meta,statistics and index, single-thread mode + /// \return MSRStatus the status of MSRStatus + Status Open(); + + /// \brief read the file, get schema meta,statistics and index, multiple-thread mode + /// \return MSRStatus the status of MSRStatus + Status Open(int n_consumer); + + /// \brief increase number of random file stream for parallel read + /// \param[in] n_new_consumers number of new file streams to be added + /// \return MSRStatus the status of MSRStatus + Status ExtendRandomFileStreams(const int n_new_consumers); + + /// \brief decrease number of random file streams for parallel read + /// \param[in] n_remove_consumers number of file streams to be removed + /// \return MSRStatus the status of MSRStatus + Status ShrinkRandomFileStreams(const int n_remove_consumers); + + /// \brief launch threads to get batches + /// \param[in] is_simple_reader trigger threads if false; do nothing if true + /// \return MSRStatus the status of MSRStatus + Status Launch(bool is_simple_reader = false); + + /// \brief aim to get the meta data + /// \return the metadata + std::shared_ptr GetShardHeader() const; + + /// \brief aim to get columns context + /// \return the columns + std::shared_ptr GetShardColumn() const; + + /// \brief get the number of shards + /// \return # of shards + int GetShardCount() const; + + /// \brief get the number of rows in database + /// \param[in] file_paths the path of ONE file, any file in dataset is fine or file list + /// \param[in] load_dataset load dataset from single file or not + /// \param[in] op smart pointer refer to ShardCategory or ShardSample object + /// \param[out] count # of rows + /// \return MSRStatus the status of MSRStatus + Status CountTotalRows(const std::vector &file_paths, bool load_dataset, + const std::shared_ptr &op, int64_t *count, const int64_t num_padded); + + /// \brief shuffle task with incremental seed + /// \return void + void ShuffleTask(); + + /// \brief get the number of rows in database + /// \return # of rows + int64_t GetNumRows() const; + + /// \brief Read the summary of row groups + /// \return the tuple of 4 elements + /// 1. Sharding ID + /// 2. Row group ID + /// 3. The row ID started in row group + /// 4. # of rows in row group + std::vector> ReadRowGroupSummary(); + + /// \brief Read 1 row group data, excluding images + /// \param[in] groupID row group ID + /// \param[in] shard_id sharding ID + /// \param[in] columns multi-columns retrieved + /// \return the tuple of 5 elements + /// 1. file name where row group is located + /// 2. Actual row group size + /// 3. Offset address of row group in file + /// 4. The list of image offset in page [startOffset, endOffset) + /// 5. The list of columns data + Status ReadRowGroupBrief(int group_id, int shard_id, const std::vector &columns, + std::shared_ptr *row_group_brief_ptr); + + /// \brief Read 1 row group data, excluding images, following an index field criteria + /// \param[in] groupID row group ID + /// \param[in] shard_id sharding ID + /// \param[in] column-value pair of criteria to fulfill + /// \param[in] columns multi-columns retrieved + /// \return the tuple of 5 elements + /// 1. file name where row group is located + /// 2. Actual row group size + /// 3. Offset address of row group in file + /// 4. The list of image offset in page [startOffset, endOffset) + /// 5. The list of columns data + Status ReadRowGroupCriteria(int group_id, int shard_id, const std::pair &criteria, + const std::vector &columns, + std::shared_ptr *row_group_brief_ptr); + + /// \brief return a batch, given that one is ready + /// \return a batch of images and image data + std::vector, json>> GetNext(); + + /// \brief return a row by id + /// \return a batch of images and image data + TASK_CONTENT GetNextById(const int64_t &task_id, const int32_t &consumer_id); + /// \brief get blob filed list + /// \return blob field list + std::pair> GetBlobFields(); + + /// \brief reset reader + /// \return null + void Reset(); + + /// \brief set flag of all-in-index + /// \return null + void SetAllInIndex(bool all_in_index) { all_in_index_ = all_in_index; } + + /// \brief get all classes + Status GetAllClasses(const std::string &category_field, std::shared_ptr> category_ptr); + + /// \brief get a read-only ptr to the sampled ids for this epoch + const std::vector *GetSampleIds(); + + /// \brief get the size of blob data + Status GetTotalBlobSize(int64_t *total_blob_size); + + /// \brief extract uncompressed data based on column list + Status UnCompressBlob(const std::vector &raw_blob_data, + std::shared_ptr>> *blob_data_ptr); + + protected: + /// \brief sqlite call back function + static int SelectCallback(void *p_data, int num_fields, char **p_fields, char **p_col_names); + + private: + /// \brief wrap up labels to json format + Status ConvertLabelToJson(const std::vector> &labels, std::shared_ptr fs, + std::shared_ptr>>> offset_ptr, int shard_id, + const std::vector &columns, + std::shared_ptr>> col_val_ptr); + + /// \brief convert json format to expected type + Status ConvertJsonValue(const std::vector &label, const std::vector &columns, + const json &schema, json *value); + + /// \brief read all rows for specified columns + Status ReadAllRowGroup(const std::vector &columns, std::shared_ptr *row_group_ptr); + + /// \brief read row meta by shard_id and sample_id + Status ReadRowGroupByShardIDAndSampleID(const std::vector &columns, const uint32_t &shard_id, + const uint32_t &sample_id, std::shared_ptr *row_group_ptr); + + /// \brief read all rows in one shard + Status ReadAllRowsInShard(int shard_id, const std::string &sql, const std::vector &columns, + std::shared_ptr>>> offset_ptr, + std::shared_ptr>> col_val_ptr); + + /// \brief initialize reader + Status Init(const std::vector &file_paths, bool load_dataset); + + /// \brief validate column list + Status CheckColumnList(const std::vector &selected_columns); + + /// \brief populate one row by task list in row-reader mode + void ConsumerByRow(int consumer_id); + + /// \brief get offset address of images within page + std::vector> GetImageOffset(int group_id, int shard_id, + const std::pair &criteria = {"", ""}); + + /// \brief get page id by category + Status GetPagesByCategory(int shard_id, const std::pair &criteria, + std::shared_ptr> *pages_ptr); + /// \brief execute sqlite query with prepare statement + Status QueryWithCriteria(sqlite3 *db, const string &sql, const string &criteria, + std::shared_ptr>> labels_ptr); + /// \brief verify the validity of dataset + Status VerifyDataset(sqlite3 **db, const string &file); + + /// \brief get column values + Status GetLabels(int page_id, int shard_id, const std::vector &columns, + const std::pair &criteria, std::shared_ptr> *labels_ptr); + + /// \brief get column values from raw data page + Status GetLabelsFromPage(int page_id, int shard_id, const std::vector &columns, + const std::pair &criteria, + std::shared_ptr> *labels_ptr); + + /// \brief create category-applied task list + Status CreateTasksByCategory(const std::shared_ptr &op); + + /// \brief create task list in row-reader mode + Status CreateTasksByRow(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief create task list in row-reader mode and lazy mode + Status CreateLazyTasksByRow(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief crate task list + Status CreateTasks(const std::vector> &row_group_summary, + const std::vector> &operators); + + /// \brief check if all specified columns are in index table + void CheckIfColumnInIndex(const std::vector &columns); + + /// \brief open multiple file handle + void FileStreamsOperator(); + + /// \brief read one row by one task + Status ConsumerOneTask(int64_t task_id, uint32_t consumer_id, std::shared_ptr *task_content_pt); + + /// \brief get labels from binary file + Status GetLabelsFromBinaryFile(int shard_id, const std::vector &columns, + const std::vector> &label_offsets, + std::shared_ptr> *labels_ptr); + + /// \brief get classes in one shard + void GetClassesInShard(sqlite3 *db, int shard_id, const std::string &sql, + std::shared_ptr> category_ptr); + + /// \brief get number of classes + int64_t GetNumClasses(const std::string &category_field); + + /// \brief get meta of header + Status GetMeta(const std::string &file_path, std::shared_ptr meta_data_ptr, + std::shared_ptr> *addresses_ptr); + + protected: + uint64_t header_size_; // header size + uint64_t page_size_; // page size + int shard_count_; // number of shards + std::shared_ptr shard_header_; // shard header + std::shared_ptr shard_column_; // shard column + + std::vector database_paths_; // sqlite handle list + std::vector file_paths_; // file paths + std::vector> file_streams_; // single-file handle list + std::vector>> file_streams_random_; // multiple-file handle list + + private: + int n_consumer_; // number of workers (threads) + std::vector selected_columns_; // columns which will be read + std::map column_schema_id_; // column-schema map + std::vector> operators_; // data operators, including shuffle, sample and category + ShardTaskList tasks_; // shard task list + std::mutex shard_locker_; // locker of shard + + // flags + bool all_in_index_ = true; // if all columns are stored in index-table + bool interrupt_ = false; // reader interrupted + + int64_t num_padded_; // number of padding samples + + // Delivery/Iterator mode begin + const std::string kThreadName = "THRD_ITER_"; // prefix of thread name + std::vector thread_set_; // thread list + int64_t num_rows_; // number of rows + int64_t total_blob_size_; // total size of blob data + std::mutex mtx_delivery_; // locker for delivery + std::condition_variable cv_delivery_; // conditional variable for delivery + std::condition_variable cv_iterator_; // conditional variable for iterator + std::atomic sample_id_position_; // index into the sample ids vector for the current sample id + std::atomic deliver_id_; // delivery ID which is picked up by iterator + // map of delivery + std::unordered_map, json>>>> delivery_map_; + // Delivery/Iterator mode end + + // all metadata in the index is not loaded during initialization + bool lazy_load_; + + // indicate shard_id : inc_count + // 0 : 15 - shard0 has 15 samples + // 1 : 41 - shard1 has 26 samples + // 2 : 58 - shard2 has 17 samples + std::vector shard_sample_count_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_READER_H_ -- 2.34.1 From 1494bfa5f8a153621620555daca4d5c924e8dc50 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:45:59 +0800 Subject: [PATCH 153/160] ADD file via upload --- .../mindrecord/include/shard_sample.h | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_sample.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_sample.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_sample.h new file mode 100644 index 00000000000..971c9a14da2 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_sample.h @@ -0,0 +1,67 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SAMPLE_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SAMPLE_H_ + +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_operator.h" +#include "minddata/mindrecord/include/shard_shuffle.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardSample : public ShardOperator { + public: + explicit ShardSample(int64_t n); + + ShardSample(int64_t num, int64_t den); + + ShardSample(int64_t num, int64_t den, int64_t par, int64_t no_of_samples = 0, int64_t offset = -1); + + ShardSample(const std::vector &indices); + + ShardSample(const std::vector &indices, uint32_t seed); + + ~ShardSample() override{}; + + Status Execute(ShardTaskList &tasks) override; + + Status UpdateTasks(ShardTaskList &tasks, int64_t taking); + + Status SufExecute(ShardTaskList &tasks) override; + + int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override; + + protected: + int64_t numerator_; + int64_t denominator_; + int64_t partition_id_; + int64_t no_of_samples_; + std::shared_ptr shuffle_op_; + std::vector nums_per_shard_; + + private: + std::vector indices_; + SamplerType sampler_type_; + int64_t offset_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SAMPLE_H_ -- 2.34.1 From 7e695dfcc3295449744b92c0ec0253d545974a87 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:46:14 +0800 Subject: [PATCH 154/160] ADD file via upload --- .../mindrecord/include/shard_schema.h | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_schema.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_schema.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_schema.h new file mode 100644 index 00000000000..9f632a375b5 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_schema.h @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SCHEMA_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SCHEMA_H_ + +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_pybind.h" +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "pybind11/pybind11.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) Schema { + public: + ~Schema() = default; + + /// \brief obtain the json schema ,its description, its block fields + /// \param[in] desc the description of the schema + /// \param[in] schema the schema's json + static std::shared_ptr Build(std::string desc, const json &schema); + + /// \brief compare two schema to judge if they are equal + /// \param b another schema to be judged + /// \return true if they are equal,false if not + bool operator==(const Schema &b) const; + + /// \brief get the schema and its description + /// \return the json format of the schema and its description + std::string GetDesc() const; + + /// \brief get the schema and its description + /// \return the json format of the schema and its description + json GetSchema() const; + + /// set the schema id + /// \param[in] id the id need to be set + void SetSchemaID(int64_t id); + + /// get the schema id + /// \return the int64 schema id + int64_t GetSchemaID() const; + + /// get the blob fields + /// \return the vector blob fields + std::vector GetBlobFields() const; + + private: + Schema() = default; + static bool ValidateNumberShape(const json &it_value); + static bool Validate(json schema); + static std::vector PopulateBlobFields(json schema); + + std::string desc_; + json schema_; + std::vector blob_fields_; + int64_t schema_id_ = -1; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SCHEMA_H_ -- 2.34.1 From 9bee22d03f50f457047717a5dcd371cfa21875e5 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:46:48 +0800 Subject: [PATCH 155/160] ADD file via upload --- .../mindrecord/include/shard_segment.h | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_segment.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_segment.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_segment.h new file mode 100644 index 00000000000..4fb1b30af61 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_segment.h @@ -0,0 +1,101 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEGMENT_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEGMENT_H_ + +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_reader.h" + +namespace mindspore { +namespace mindrecord { +using CATEGORY_INFO = std::vector>; +using PAGES = std::vector, json>>; +using PAGES_LOAD = std::vector, pybind11::object>>; + +class __attribute__((visibility("default"))) ShardSegment : public ShardReader { + public: + ShardSegment(); + + ~ShardSegment() override = default; + + /// \brief Get candidate category fields + /// \return a list of fields names which are the candidates of category + Status GetCategoryFields(std::shared_ptr> *fields_ptr); + + /// \brief Set category field + /// \param[in] category_field category name + /// \return true if category name is existed + Status SetCategoryField(std::string category_field); + + /// \brief Thread-safe implementation of ReadCategoryInfo + /// \return statistics data in json format with 2 field: "key" and "categories". + /// The value of "categories" is a list. Each Element in list is {count, id， name} + /// count: count of images in category + /// id: internal unique identification, persistent + /// name: category name + /// example: + /// { "key": "label", + /// "categories": [ { "count": 3, "id": 0, "name": "sport", }, + /// { "count": 3, "id": 1, "name": "finance", } ] } + Status ReadCategoryInfo(std::shared_ptr *category_ptr); + + /// \brief Thread-safe implementation of ReadAtPageById + /// \param[in] category_id category ID + /// \param[in] page_no page number + /// \param[in] n_rows_of_page rows number in one page + /// \return images array, image is a vector of uint8_t + Status ReadAtPageById(int64_t category_id, int64_t page_no, int64_t n_rows_of_page, + std::shared_ptr>> *page_ptr); + + /// \brief Thread-safe implementation of ReadAtPageByName + /// \param[in] category_name category Name + /// \param[in] page_no page number + /// \param[in] n_rows_of_page rows number in one page + /// \return images array, image is a vector of uint8_t + Status ReadAtPageByName(std::string category_name, int64_t page_no, int64_t n_rows_of_page, + std::shared_ptr>> *pages_ptr); + + Status ReadAllAtPageById(int64_t category_id, int64_t page_no, int64_t n_rows_of_page, + std::shared_ptr *pages_ptr); + + Status ReadAllAtPageByName(std::string category_name, int64_t page_no, int64_t n_rows_of_page, + std::shared_ptr *pages_ptr); + + std::pair> GetBlobFields(); + + private: + Status WrapCategoryInfo(std::shared_ptr *category_info_ptr); + + std::string ToJsonForCategory(const std::vector> &tri_vec); + + std::string CleanUp(std::string fieldName); + + Status PackImages(int group_id, int shard_id, std::vector offset, + std::shared_ptr> *images_ptr); + + std::vector candidate_category_fields_; + std::string current_category_field_; + const uint32_t kStartFieldId = 9; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEGMENT_H_ -- 2.34.1 From b32842d0dc91406fe7eda2567ae1ca7c02accfd2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:47:10 +0800 Subject: [PATCH 156/160] ADD file via upload --- .../include/shard_sequential_sample.h | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_sequential_sample.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_sequential_sample.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_sequential_sample.h new file mode 100644 index 00000000000..9f86621f988 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_sequential_sample.h @@ -0,0 +1,48 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEQUENTIAL_SAMPLE_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEQUENTIAL_SAMPLE_H_ + +#include +#include +#include +#include +#include "minddata/mindrecord/include/shard_sample.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardSequentialSample : public ShardSample { + public: + ShardSequentialSample(int64_t n, int64_t offset); + + ShardSequentialSample(float per, float per_offset); + + ~ShardSequentialSample() override{}; + + Status Execute(ShardTaskList &tasks) override; + + int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override; + + private: + int64_t offset_; + float per_; + float per_offset_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SEQUENTIAL_SAMPLE_H_ -- 2.34.1 From 79d95999d7bc4fa7942973dbf6fa6f79ddbfb0fb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:47:27 +0800 Subject: [PATCH 157/160] ADD file via upload --- .../mindrecord/include/shard_shuffle.h | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_shuffle.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_shuffle.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_shuffle.h new file mode 100644 index 00000000000..b14d975e581 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_shuffle.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_ + +#include +#include "minddata/mindrecord/include/shard_operator.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardShuffle : public ShardOperator { + public: + explicit ShardShuffle(uint32_t seed = 0, ShuffleType shuffle_type = kShuffleCategory); + + ShardShuffle(uint32_t seed, int64_t no_of_samples, bool replacement, bool reshuffle_each_epoch, + ShuffleType shuffle_type = kShuffleSample); + + ~ShardShuffle() override{}; + + Status Execute(ShardTaskList &tasks) override; + + int64_t GetNumSamples(int64_t dataset_size, int64_t num_classes) override; + + private: + // Private helper function + Status CategoryShuffle(ShardTaskList &tasks); + + // Keep the file sequence the same but shuffle the data within each file + Status ShuffleInfile(ShardTaskList &tasks); + + // Shuffle the file sequence but keep the order of data within each file + Status ShuffleFiles(ShardTaskList &tasks); + + uint32_t shuffle_seed_; + int64_t no_of_samples_; + bool replacement_; + bool reshuffle_each_epoch_; + ShuffleType shuffle_type_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_SHUFFLE_H_ -- 2.34.1 From cf50c75accecf78fb67a40c6c33a260d7197b953 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:47:42 +0800 Subject: [PATCH 158/160] ADD file via upload --- .../mindrecord/include/shard_statistics.h | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_statistics.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_statistics.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_statistics.h new file mode 100644 index 00000000000..910ff9cff1a --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_statistics.h @@ -0,0 +1,82 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_STATISTICS_H +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_STATISTICS_H + +#include +#include +#include +#include +#include + +#include "minddata/mindrecord/include/common/shard_pybind.h" +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "pybind11/pybind11.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) Statistics { + public: + /// \brief save the statistic and its description + /// \param[in] desc the statistic's description + /// \param[in] statistics the statistic needs to be saved + static std::shared_ptr Build(std::string desc, const json &statistics); + + ~Statistics() = default; + + /// \brief compare two statistics to judge if they are equal + /// \param b another statistics to be judged + /// \return true if they are equal,false if not + bool operator==(const Statistics &b) const; + + /// \brief get the description + /// \return the description + std::string GetDesc() const; + + /// \brief get the statistic + /// \return json format of the statistic + json GetStatistics() const; + + /// \brief decode the bson statistics to json + /// \param[in] encodedStatistics the bson type of statistics + /// \return json type of statistic + void SetStatisticsID(int64_t id); + + /// \brief get the statistics id + /// \return the int64 statistics id + int64_t GetStatisticsID() const; + + private: + /// \brief validate the statistic + /// \return true / false + static bool Validate(const json &statistics); + + static bool LevelRecursive(json level); + + Statistics() = default; + + std::string desc_; + json statistics_; + int64_t statistics_id_ = -1; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_STATISTICS_H -- 2.34.1 From 1a180414a0faa2a0c288f0335a1c22ce563a9c23 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:47:57 +0800 Subject: [PATCH 159/160] ADD file via upload --- .../mindrecord/include/shard_task_list.h | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_task_list.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_task_list.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_task_list.h new file mode 100644 index 00000000000..e77c4624762 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_task_list.h @@ -0,0 +1,132 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_TASK_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_TASK_H_ + +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" + +namespace mindspore { +namespace mindrecord { + +// The data struct is as below: +// 1. TaskType: kCommonTask / kPaddedTask +// 2. std::tuple : shard_id, group_id(fast load) / sample_id(lazy load) +// 3. std::vector, json>> : [blob_start, blob_end], scalar_variable_fields +using ShardTask = std::tuple, std::vector, json>; + +class __attribute__((visibility("default"))) ShardTaskList { + public: + ShardTaskList(); + + ShardTaskList(const ShardTaskList &task); // copy construction + + ShardTaskList &operator=(const ShardTaskList &task); // assignment operator + + ~ShardTaskList() = default; + + void InitSampleIds(); + + static void TaskListSwap(ShardTaskList &orig_tasks, ShardTaskList &new_tasks); + + // Assigns the task based on task id + inline void AssignTask(ShardTaskList &sourceTasks, int64_t id); + + inline void InsertTask(TaskType task_type, int shard_id, int group_id, const std::vector &offset, + const json &label); + + inline void InsertTask(const int64_t &i, TaskType task_type, int shard_id, int group_id, + const std::vector &offset, const json &label); + + inline void InsertTask(ShardTask task); + + inline void InsertTask(const int64_t &i, ShardTask task); + + void MakePerm(); + + inline void InsertSampleId(int64_t id); + + void PopBack(); + + int64_t Size() const; + + int64_t SizeOfRows() const; + + ShardTask &GetTaskByID(int64_t id); + + ShardTask &GetRandomTask(); + + int64_t GetTaskSampleByID(int64_t id); + + int64_t GetRandomTaskID(); + + static ShardTaskList Combine(std::vector &category_tasks, bool replacement, int64_t num_elements, + int64_t num_samples); + + inline void ResizeTask(const int64_t &size); + + uint32_t categories; + + std::vector permutation_; // A list of ints used for shuffling sample ids + + std::vector sample_ids_; // The list of actual ids that were sampled + + std::vector task_list_; // The full list of tasks +}; + +inline void ShardTaskList::AssignTask(ShardTaskList &sourceTasks, int64_t id) { + // Insert the sample id from the source into ourself by indexing at id position. + // Important: The task list itself does not change. + int64_t sample_id = sourceTasks.GetTaskSampleByID(id); + MS_LOG(DEBUG) << "Insert sample id (" << sample_id << ") into task list from source task position: " << id; + sample_ids_.push_back(sample_id); +} + +inline void ShardTaskList::InsertTask(TaskType task_type, int shard_id, int group_id, + const std::vector &offset, const json &label) { + MS_LOG(DEBUG) << "Insert task into task list, shard_id: " << shard_id << ", group_id: " << group_id + << ", label: " << label.dump() << ", size of task_list_: " << task_list_.size() << "."; + task_list_.emplace_back(task_type, std::make_tuple(shard_id, group_id), offset, label); +} + +inline void ShardTaskList::InsertTask(const int64_t &i, TaskType task_type, int shard_id, int group_id, + const std::vector &offset, const json &label) { + MS_LOG(DEBUG) << "Insert task into task list, shard_id: " << shard_id << ", group_id: " << group_id + << ", label: " << label.dump() << ", size of task_list_: " << task_list_.size() << "."; + task_list_[i] = {task_type, std::make_tuple(shard_id, group_id), offset, label}; +} + +inline void ShardTaskList::InsertTask(ShardTask task) { + MS_LOG(DEBUG) << "Insert task into task list, shard_id: " << std::get<0>(std::get<1>(task)) + << ", group_id: " << std::get<1>(std::get<1>(task)) << ", label: " << std::get<3>(task).dump() + << ", size of task_list_: " << task_list_.size() << "."; + + task_list_.push_back(std::move(task)); +} + +inline void ShardTaskList::InsertTask(const int64_t &i, ShardTask task) { task_list_[i] = std::move(task); } + +inline void ShardTaskList::ResizeTask(const int64_t &size) { task_list_.resize(size); } +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_TASK_H_ -- 2.34.1 From 0f4c1ae10bb0946f271a17cdf023206da92273b2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Mon, 17 Oct 2022 22:48:11 +0800 Subject: [PATCH 160/160] ADD file via upload --- .../mindrecord/include/shard_writer.h | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 mindspore2022/ccsrc/minddata/mindrecord/include/shard_writer.h diff --git a/mindspore2022/ccsrc/minddata/mindrecord/include/shard_writer.h b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_writer.h new file mode 100644 index 00000000000..ee7cf2e2792 --- /dev/null +++ b/mindspore2022/ccsrc/minddata/mindrecord/include/shard_writer.h @@ -0,0 +1,256 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_WRITER_H_ +#define MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_WRITER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "minddata/mindrecord/include/common/shard_utils.h" +#include "minddata/mindrecord/include/shard_column.h" +#include "minddata/mindrecord/include/shard_error.h" +#include "minddata/mindrecord/include/shard_header.h" +#include "minddata/mindrecord/include/shard_index.h" +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace mindrecord { +class __attribute__((visibility("default"))) ShardWriter { + public: + ShardWriter(); + + ~ShardWriter(); + + /// \brief Open file at the beginning 在开头打开文件 + /// \param[in] paths the file names list 文件名列表 + /// \param[in] append new data at the end of file if true, otherwise try to overwrite file 如果为 true，则文件末尾的新数据，否则尝试覆盖文件 + /// \param[in] overwrite a file with the same name if true 具有相同名称的文件（如果为 true） + /// \return Status + Status Open(const std::vector &paths, bool append = false, bool overwrite = false); + + /// \brief Open file at the ending 在结尾处打开文件 + /// \param[in] paths the file names list 文件名列表 + /// \return MSRStatus the status of MSRStatus MSR状态 + Status OpenForAppend(const std::string &path); + + /// \brief Write header to disk 将标头写入磁盘 + /// \return MSRStatus the status of MSRStatus MSR状态 + Status Commit(); + + /// \brief Set file size 设置文件大小 + /// \param[in] header_size the size of header, only (1< header_data); + + /// \brief write raw data by group size 按组大小写入原始数据 + /// \param[in] raw_data the vector of raw json data, vector format 原始 json 数据的向量，向量格式 + /// \param[in] blob_data the vector of image data 图像数据的向量 + /// \param[in] sign validate data or not + /// \return MSRStatus the status of MSRStatus to judge if write successfully MSRStatus 判断写入是否成功的 MSRStatus 状态 + Status WriteRawData(std::map> &raw_data, vector> &blob_data, + bool sign = true, bool parallel_writer = false); + + /// \brief write raw data by group size for call from python 按组大小写入原始数据，以便从 python 调用 + /// \param[in] raw_data the vector of raw json data, python-handle format 原始 json 数据的向量，python 句柄格式 + /// \param[in] blob_data the vector of blob json data, python-handle format blob json 数据的向量，python-handle 格式 + /// \param[in] sign validate data or not 验证数据与否 + /// \return MSRStatus the status of MSRStatus to judge if write successfully MSRStatus 判断写入是否成功的 MSRStatus 状态 + Status WriteRawData(std::map> &raw_data, + std::map> &blob_data, bool sign = true, + bool parallel_writer = false); + + Status MergeBlobData(const std::vector &blob_fields, + const std::map>> &row_bin_data, + std::shared_ptr> *output); + + static Status Initialize(const std::unique_ptr *writer_ptr, const std::vector &file_names); + + private: + /// \brief write shard header data to disk 将分片头数据写入磁盘 + Status WriteShardHeader(); + + /// \brief erase error data 擦除错误数据 + void DeleteErrorData(std::map> &raw_data, std::vector> &blob_data); + + /// \brief populate error data 填充错误数据 + void PopulateMutexErrorData(const int &row, const std::string &message, std::map &err_raw_data); + + /// \brief check data 检查数据 + void CheckSliceData(int start_row, int end_row, json schema, const std::vector &sub_raw_data, + std::map &err_raw_data); + + /// \brief write shard header data to disk 将分片头数据写入磁盘 + Status ValidateRawData(std::map> &raw_data, std::vector> &blob_data, + bool sign, std::shared_ptr> *count_ptr); + + /// \brief fill data array in multiple thread run 在多线程运行中填充数据数组 + void FillArray(int start, int end, std::map> &raw_data, + std::vector> &bin_data); + + /// \brief serialized raw data 序列化的原始数据 + Status SerializeRawData(std::map> &raw_data, std::vector> &bin_data, + uint32_t row_count); + + /// \brief write all data parallel 并行写入所有数据 + Status ParallelWriteData(const std::vector> &blob_data, + const std::vector> &bin_raw_data); + + /// \brief write data shard by shard 逐个分片写入数据分片 + Status WriteByShard(int shard_id, int start_row, int end_row, const std::vector> &blob_data, + const std::vector> &bin_raw_data); + + /// \brief break image data up into multiple row groups 将图像数据分解为多个行组 + Status CutRowGroup(int start_row, int end_row, const std::vector> &blob_data, + std::vector> &rows_in_group, const std::shared_ptr &last_raw_page, + const std::shared_ptr &last_blob_page); + + /// \brief append partial blob data to previous page 将部分 Blob 数据追加到上一页 + Status AppendBlobPage(const int &shard_id, const std::vector> &blob_data, + const std::vector> &rows_in_group, + const std::shared_ptr &last_blob_page); + + /// \brief write new blob data page to disk 将新的 Blob 数据页写入磁盘 + Status NewBlobPage(const int &shard_id, const std::vector> &blob_data, + const std::vector> &rows_in_group, + const std::shared_ptr &last_blob_page); + + /// \brief shift last row group to next raw page for new appending 将最后一行组移动到下一个原始页面以进行新的追加 + Status ShiftRawPage(const int &shard_id, const std::vector> &rows_in_group, + std::shared_ptr &last_raw_page); + + /// \brief write raw data page to disk 将原始数据页写入磁盘 + Status WriteRawPage(const int &shard_id, const std::vector> &rows_in_group, + std::shared_ptr &last_raw_page, const std::vector> &bin_raw_data); + + /// \brief generate empty raw data page 生成空的原始数据页面 + Status EmptyRawPage(const int &shard_id, std::shared_ptr &last_raw_page); + + /// \brief append a row group at the end of raw page 在原始页面末尾追加行组 + Status AppendRawPage(const int &shard_id, const std::vector> &rows_in_group, const int &chunk_id, + int &last_row_groupId, std::shared_ptr last_raw_page, + const std::vector> &bin_raw_data); + + /// \brief write blob chunk to disk 将 blob 块写入磁盘 + Status FlushBlobChunk(const std::shared_ptr &out, const std::vector> &blob_data, + const std::pair &blob_row); + + /// \brief write raw chunk to disk 将原始块写入磁盘 + Status FlushRawChunk(const std::shared_ptr &out, const std::vector> &rows_in_group, + const int &chunk_id, const std::vector> &bin_raw_data); + + /// \brief break up into tasks by shard 按分片分解为任务 + std::vector> BreakIntoShards(); + + /// \brief calculate raw data size row by row 逐行计算原始数据大小 + Status SetRawDataSize(const std::vector> &bin_raw_data); + + /// \brief calculate blob data size row by row 逐行计算 Blob 数据大小 + Status SetBlobDataSize(const std::vector> &blob_data); + + /// \brief populate last raw page pointer 填充最后一个原始页面指针 + Status SetLastRawPage(const int &shard_id, std::shared_ptr &last_raw_page); + + /// \brief populate last blob page pointer 填充最后一个 blob 页指针 + Status SetLastBlobPage(const int &shard_id, std::shared_ptr &last_blob_page); + + /// \brief check the data by schema 按架构检查数据 + Status CheckData(const std::map> &raw_data); + + /// \brief check the data and type 检查数据和类型 + Status CheckDataTypeAndValue(const std::string &key, const json &value, const json &data, const int &i, + std::map &err_raw_data); + + /// \brief Lock writer and save pages info 锁定编写器并保存页面信息 + Status LockWriter(bool parallel_writer, std::unique_ptr *fd_ptr); + + /// \brief Unlock writer and save pages info 解锁作家并保存页面信息 + Status UnlockWriter(int fd, bool parallel_writer = false); + + /// \brief Check raw data before writing 写入前检查原始数据 + Status WriteRawDataPreCheck(std::map> &raw_data, vector> &blob_data, + bool sign, int *schema_count, int *row_count); + + /// \brief Get full path from file name 从文件名获取完整路径 + Status GetFullPathFromFileName(const std::vector &paths); + + /// \brief Open files 打开文件 + Status OpenDataFiles(bool append, bool overwrite); + + /// \brief Remove lock file 删除锁定文件 + Status RemoveLockFile(); + + /// \brief Remove lock file 删除锁定文件 + Status InitLockFile(); + + private: + const std::string kLockFileSuffix = "_Locker"; + const std::string kPageFileSuffix = "_Pages"; + std::string lock_file_; // lock file for parallel run 锁定文件以进行并行运行 + std::string pages_file_; // temporary file of pages info for parallel run 用于并行运行的页面信息的临时文件 + + int shard_count_; // number of files 文件数 + uint64_t header_size_; // header size 页眉大小 + uint64_t page_size_; // page size 页面大小 + uint32_t row_count_; // count of rows 行数 + uint32_t schema_count_; // count of schemas 架构计数 + + std::vector raw_data_size_; // Raw data size 原始数据大小 + std::vector blob_data_size_; // Blob data size Blob 数据大小 + + std::vector file_paths_; // file paths 文件路径 + std::vector> file_streams_; // file handles 文件句柄 + std::shared_ptr shard_header_; // shard header 分片头 + std::shared_ptr shard_column_; // shard columns 分片列 + + std::map> err_mg_; // used for storing error raw_data info 用于存储错误raw_data信息 + + std::mutex check_mutex_; // mutex for data check 用于数据检查的互斥锁 + std::atomic flag_{false}; + std::atomic compression_size_; +}; +} // namespace mindrecord +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_MINDDATA_MINDRECORD_INCLUDE_SHARD_WRITER_H_ -- 2.34.1