diff --git a/mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py b/mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py index 3aef08133e271a32252bc8540a16103bb4e78a52..38e88646d0d5511298402d52f26890c5c339b863 100644 --- a/mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py +++ b/mindspore/nn/probability/bnn_layers/bnn_cell_wrapper.py @@ -65,9 +65,9 @@ class WithBNNLossCell: """ def __init__(self, backbone, loss_fn, dnn_factor=1, bnn_factor=1): - if not isinstance(dnn_factor, (int, float)): + if type(dnn_factor) not in [int, float]: raise TypeError('The type of `dnn_factor` should be `int` or `float`') - if not isinstance(bnn_factor, (int, float)): + if type(bnn_factor) not in [int, float]: raise TypeError('The type of `bnn_factor` should be `int` or `float`') self.backbone = backbone diff --git a/mindspore/nn/probability/bnn_layers/conv_variational.py b/mindspore/nn/probability/bnn_layers/conv_variational.py index 6389ea9b2faeb2934c3a8af5fade435b095345cc..3e85b879d88145de437c1210ac20b1cd1d1fe97f 100644 --- a/mindspore/nn/probability/bnn_layers/conv_variational.py +++ b/mindspore/nn/probability/bnn_layers/conv_variational.py @@ -179,7 +179,7 @@ class ConvReparam(_ConvVariational): Args: in_channels (int): The number of input channel :math:`C_{in}`. out_channels (int): The number of output channel :math:`C_{out}`. - kernel_size (Union[int, tuple[int]]): The data type is int or + kernel_size (Union[int, tuple[int]]): The data type is int or tuple with 2 integers. Specifies the height and width of the 2D convolution window. Single int means the value if for both height and width of the kernel. A tuple of 2 ints means the diff --git a/mindspore/nn/probability/bnn_layers/layer_distribution.py b/mindspore/nn/probability/bnn_layers/layer_distribution.py index 262af94403226345cc8ba5d31860267f4fc1c35d..f43f628793623c0174a2dc7f241b528d0645058f 100644 --- a/mindspore/nn/probability/bnn_layers/layer_distribution.py +++ b/mindspore/nn/probability/bnn_layers/layer_distribution.py @@ -75,19 +75,19 @@ class NormalPosterior(Cell): if not isinstance(name, str): raise TypeError('The type of `name` should be `str`') - if not isinstance(shape, (tuple, list)): + if type(shape) not in [tuple, list]: raise TypeError('The type of `shape` should be `tuple` or `list`') - if not isinstance(loc_mean, (int, float)): + if type(loc_mean) not in [int, float]: raise TypeError('The type of `loc_mean` should be `int` or `float`') - if not isinstance(untransformed_scale_mean, (int, float)): + if type(untransformed_scale_mean) not in [int, float]: raise TypeError('The type of `untransformed_scale_mean` should be `int` or `float`') - if not (isinstance(loc_std, (int, float)) and loc_std >= 0): + if not (type(loc_std) in [int, float] and loc_std >= 0): raise TypeError('The type of `loc_std` should be `int` or `float` and its value should > 0') - if not (isinstance(untransformed_scale_std, (int, float)) and untransformed_scale_std >= 0): + if not (type(untransformed_scale_std) in [int, float] and untransformed_scale_std >= 0): raise TypeError('The type of `untransformed_scale_std` should be `int` or `float` and ' 'its value should > 0') diff --git a/mindspore/nn/probability/transforms/transform_bnn.py b/mindspore/nn/probability/transforms/transform_bnn.py index 3853c48b2395d20a254afacb3b752dd308bc3b50..49e8f7777346b2734eb5cd38c5cd33c9a5223d66 100644 --- a/mindspore/nn/probability/transforms/transform_bnn.py +++ b/mindspore/nn/probability/transforms/transform_bnn.py @@ -61,9 +61,9 @@ class TransformToBNN: """ def __init__(self, trainable_dnn, dnn_factor=1, bnn_factor=1): - if not isinstance(dnn_factor, (int, float)): + if type(dnn_factor) not in [int, float]: raise TypeError('The type of `dnn_factor` should be `int` or `float`') - if not isinstance(bnn_factor, (int, float)): + if type(bnn_factor) not in [int, float]: raise TypeError('The type of `bnn_factor` should be `int` or `float`') net_with_loss = trainable_dnn.network diff --git a/tests/st/probability/dataset.py b/tests/st/probability/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cef69734839d1efeefefcc9dc31cd620c451917a --- /dev/null +++ b/tests/st/probability/dataset.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Produce the dataset +""" + +import mindspore.dataset as ds +import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.transforms.c_transforms as C +from mindspore.dataset.transforms.vision import Inter +from mindspore.common import dtype as mstype + + +def create_dataset(data_path, batch_size=32, repeat_size=1, + num_parallel_workers=1): + """ + create dataset for train or test + """ + # define dataset + mnist_ds = ds.MnistDataset(data_path) + + resize_height, resize_width = 32, 32 + rescale = 1.0 / 255.0 + shift = 0.0 + rescale_nml = 1 / 0.3081 + shift_nml = -1 * 0.1307 / 0.3081 + + # define map operations + resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) # Bilinear mode + rescale_nml_op = CV.Rescale(rescale_nml, shift_nml) + rescale_op = CV.Rescale(rescale, shift) + hwc2chw_op = CV.HWC2CHW() + type_cast_op = C.TypeCast(mstype.int32) + + # apply map operations on images + mnist_ds = mnist_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=resize_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=rescale_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=rescale_nml_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=hwc2chw_op, num_parallel_workers=num_parallel_workers) + + # apply DatasetOps + buffer_size = 10000 + mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) # 10000 as in LeNet train script + mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) + mnist_ds = mnist_ds.repeat(repeat_size) + + return mnist_ds diff --git a/tests/st/probability/test_bnn_layer.py b/tests/st/probability/test_bnn_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..b135d0bf085630fff3145b58ae5211fcd7ec2772 --- /dev/null +++ b/tests/st/probability/test_bnn_layer.py @@ -0,0 +1,145 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test bnn layers""" + +import numpy as np +from mindspore import Tensor +from mindspore.common.initializer import TruncatedNormal +import mindspore.nn as nn +from mindspore.nn import TrainOneStepCell +from mindspore.nn.probability import bnn_layers +from mindspore.ops import operations as P +from mindspore import context +from dataset import create_dataset + +context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target="GPU") + + +def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): + """weight initial for conv layer""" + weight = weight_variable() + return nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, padding=padding, + weight_init=weight, has_bias=False, pad_mode="valid") + + +def fc_with_initialize(input_channels, out_channels): + """weight initial for fc layer""" + weight = weight_variable() + bias = weight_variable() + return nn.Dense(input_channels, out_channels, weight, bias) + + +def weight_variable(): + """weight initial""" + return TruncatedNormal(0.02) + + +class BNNLeNet5(nn.Cell): + """ + bayesian Lenet network + + Args: + num_class (int): Num classes. Default: 10. + + Returns: + Tensor, output tensor + Examples: + >>> BNNLeNet5(num_class=10) + + """ + def __init__(self, num_class=10): + super(BNNLeNet5, self).__init__() + self.num_class = num_class + self.conv1 = bnn_layers.ConvReparam(1, 6, 5, stride=1, padding=0, has_bias=False, pad_mode="valid") + self.conv2 = conv(6, 16, 5) + self.fc1 = bnn_layers.DenseReparam(16 * 5 * 5, 120) + self.fc2 = fc_with_initialize(120, 84) + self.fc3 = fc_with_initialize(84, self.num_class) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + self.reshape = P.Reshape() + + def construct(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.relu(x) + x = self.fc3(x) + return x + + +def train_model(train_net, net, dataset): + accs = [] + loss_sum = 0 + for _, data in enumerate(dataset.create_dict_iterator()): + train_x = Tensor(data['image'].astype(np.float32)) + label = Tensor(data['label'].astype(np.int32)) + loss = train_net(train_x, label) + output = net(train_x) + log_output = P.LogSoftmax(axis=1)(output) + acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy()) + accs.append(acc) + loss_sum += loss.asnumpy() + + loss_sum = loss_sum / len(accs) + acc_mean = np.mean(accs) + return loss_sum, acc_mean + + +def validate_model(net, dataset): + accs = [] + for _, data in enumerate(dataset.create_dict_iterator()): + train_x = Tensor(data['image'].astype(np.float32)) + label = Tensor(data['label'].astype(np.int32)) + output = net(train_x) + log_output = P.LogSoftmax(axis=1)(output) + acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy()) + accs.append(acc) + + acc_mean = np.mean(accs) + return acc_mean + + +if __name__ == "__main__": + network = BNNLeNet5() + + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + optimizer = nn.AdamWeightDecay(params=network.trainable_params(), learning_rate=0.0001) + + net_with_loss = bnn_layers.WithBNNLossCell(network, criterion, 60000, 0.000001) + train_bnn_network = TrainOneStepCell(net_with_loss, optimizer) + train_bnn_network.set_train() + + train_set = create_dataset('/home/workspace/mindspore_dataset/mnist_data/train', 64, 1) + test_set = create_dataset('/home/workspace/mindspore_dataset/mnist_data/test', 64, 1) + + epoch = 100 + + for i in range(epoch): + train_loss, train_acc = train_model(train_bnn_network, network, test_set) + + valid_acc = validate_model(network, test_set) + + print('Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tvalidation Accuracy: {:.4f}'.format( + i, train_loss, train_acc, valid_acc)) diff --git a/tests/st/probability/test_transform_bnn_layer.py b/tests/st/probability/test_transform_bnn_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..590fee8e8111f6684916ee3243b0c505dd3d7ce3 --- /dev/null +++ b/tests/st/probability/test_transform_bnn_layer.py @@ -0,0 +1,150 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test transform_to_bnn_layer""" + +import numpy as np +from mindspore import Tensor +from mindspore.common.initializer import TruncatedNormal +import mindspore.nn as nn +from mindspore.nn import TrainOneStepCell, WithLossCell +from mindspore.nn.probability import transforms, bnn_layers +from mindspore.ops import operations as P +from mindspore import context +from dataset import create_dataset + + +context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target="GPU") + + +def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): + """weight initial for conv layer""" + weight = weight_variable() + return nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, padding=padding, + weight_init=weight, has_bias=False, pad_mode="valid") + + +def fc_with_initialize(input_channels, out_channels): + """weight initial for fc layer""" + weight = weight_variable() + bias = weight_variable() + return nn.Dense(input_channels, out_channels, weight, bias) + + +def weight_variable(): + """weight initial""" + return TruncatedNormal(0.02) + + +class LeNet5(nn.Cell): + """ + Lenet network + + Args: + num_class (int): Num classes. Default: 10. + + Returns: + Tensor, output tensor + Examples: + >>> LeNet5(num_class=10) + + """ + def __init__(self, num_class=10): + super(LeNet5, self).__init__() + self.num_class = num_class + self.conv1 = conv(1, 6, 5) + self.conv2 = conv(6, 16, 5) + self.fc1 = fc_with_initialize(16 * 5 * 5, 120) + self.fc2 = fc_with_initialize(120, 84) + self.fc3 = fc_with_initialize(84, self.num_class) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + self.reshape = P.Reshape() + + def construct(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.relu(x) + x = self.fc3(x) + return x + + +def train_model(train_net, net, dataset): + accs = [] + loss_sum = 0 + for _, data in enumerate(dataset.create_dict_iterator()): + train_x = Tensor(data['image'].astype(np.float32)) + label = Tensor(data['label'].astype(np.int32)) + loss = train_net(train_x, label) + output = net(train_x) + log_output = P.LogSoftmax(axis=1)(output) + acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy()) + accs.append(acc) + loss_sum += loss.asnumpy() + + loss_sum = loss_sum / len(accs) + acc_mean = np.mean(accs) + return loss_sum, acc_mean + + +def validate_model(net, dataset): + accs = [] + for _, data in enumerate(dataset.create_dict_iterator()): + train_x = Tensor(data['image'].astype(np.float32)) + label = Tensor(data['label'].astype(np.int32)) + output = net(train_x) + log_output = P.LogSoftmax(axis=1)(output) + acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy()) + accs.append(acc) + + acc_mean = np.mean(accs) + return acc_mean + + +if __name__ == "__main__": + network = LeNet5() + + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + optimizer = nn.AdamWeightDecay(params=network.trainable_params(), learning_rate=0.0001) + + net_with_loss = WithLossCell(network, criterion) + train_network = TrainOneStepCell(net_with_loss, optimizer) + + bnn_transformer = transforms.TransformToBNN(train_network, 60000, 0.000001) + train_bnn_network = bnn_transformer.transform_to_bnn_layer(nn.Conv2d, bnn_layers.ConvReparam) + # train_bnn_network = bnn_transformer.transform_to_bnn_layer(nn.Dense, bnn_layers.DenseReparam) + train_bnn_network.set_train() + + train_set = create_dataset('/home/workspace/mindspore_dataset/mnist_data/train', 64, 1) + test_set = create_dataset('/home/workspace/mindspore_dataset/mnist_data/test', 64, 1) + + epoch = 100 + + for i in range(epoch): + train_loss, train_acc = train_model(train_bnn_network, network, test_set) + + valid_acc = validate_model(network, test_set) + + print('Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tvalidation Accuracy: {:.4f}'.format( + i, train_loss, train_acc, valid_acc)) diff --git a/tests/st/probability/test_transform_bnn_model.py b/tests/st/probability/test_transform_bnn_model.py new file mode 100644 index 0000000000000000000000000000000000000000..015a1f41d761625f1ef1d5c13376c8c02cfa3235 --- /dev/null +++ b/tests/st/probability/test_transform_bnn_model.py @@ -0,0 +1,149 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""test transform_to_bnn_model""" +import numpy as np +from mindspore import Tensor +from mindspore.common.initializer import TruncatedNormal +import mindspore.nn as nn +from mindspore.nn import WithLossCell, TrainOneStepCell +from mindspore.nn.probability import transforms +from mindspore.ops import operations as P +from mindspore import context +from dataset import create_dataset + + +context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target="GPU") + + +def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): + """weight initial for conv layer""" + weight = weight_variable() + return nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, padding=padding, + weight_init=weight, has_bias=False, pad_mode="valid") + + +def fc_with_initialize(input_channels, out_channels): + """weight initial for fc layer""" + weight = weight_variable() + bias = weight_variable() + return nn.Dense(input_channels, out_channels, weight, bias) + + +def weight_variable(): + """weight initial""" + return TruncatedNormal(0.02) + + +class LeNet5(nn.Cell): + """ + Lenet network + + Args: + num_class (int): Num classes. Default: 10. + + Returns: + Tensor, output tensor + Examples: + >>> LeNet5(num_class=10) + + """ + def __init__(self, num_class=10): + super(LeNet5, self).__init__() + self.num_class = num_class + self.conv1 = conv(1, 6, 5) + self.conv2 = conv(6, 16, 5) + self.fc1 = fc_with_initialize(16 * 5 * 5, 120) + self.fc2 = fc_with_initialize(120, 84) + self.fc3 = fc_with_initialize(84, self.num_class) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + self.reshape = P.Reshape() + + def construct(self, x): + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + x = self.relu(x) + x = self.fc3(x) + return x + + +def train_model(train_net, net, dataset): + accs = [] + loss_sum = 0 + for _, data in enumerate(dataset.create_dict_iterator()): + train_x = Tensor(data['image'].astype(np.float32)) + label = Tensor(data['label'].astype(np.int32)) + loss = train_net(train_x, label) + output = net(train_x) + log_output = P.LogSoftmax(axis=1)(output) + acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy()) + accs.append(acc) + loss_sum += loss.asnumpy() + + loss_sum = loss_sum / len(accs) + acc_mean = np.mean(accs) + return loss_sum, acc_mean + + +def validate_model(net, dataset): + accs = [] + for _, data in enumerate(dataset.create_dict_iterator()): + train_x = Tensor(data['image'].astype(np.float32)) + label = Tensor(data['label'].astype(np.int32)) + output = net(train_x) + log_output = P.LogSoftmax(axis=1)(output) + acc = np.mean(log_output.asnumpy().argmax(axis=1) == label.asnumpy()) + accs.append(acc) + + acc_mean = np.mean(accs) + return acc_mean + + +if __name__ == "__main__": + network = LeNet5() + + criterion = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + optimizer = nn.AdamWeightDecay(params=network.trainable_params(), learning_rate=0.0001) + + net_with_loss = WithLossCell(network, criterion) + train_network = TrainOneStepCell(net_with_loss, optimizer) + + bnn_transformer = transforms.TransformToBNN(train_network, 60000, 0.000001) + + train_bnn_network = bnn_transformer.transform_to_bnn_model() + train_bnn_network.set_train() + + train_set = create_dataset('/home/workspace/mindspore_dataset/mnist_data/train', 64, 1) + test_set = create_dataset('/home/workspace/mindspore_dataset/mnist_data/test', 64, 1) + + epoch = 500 + + for i in range(epoch): + train_loss, train_acc = train_model(train_bnn_network, network, test_set) + + valid_acc = validate_model(network, test_set) + + print('Epoch: {} \tTraining Loss: {:.4f} \tTraining Accuracy: {:.4f} \tvalidation Accuracy: {:.4f}'.format( + i, train_loss, train_acc, valid_acc))