Using sum efficiency, and the per layer entropy mean

functions.py

@@ -49,9 +49,9 @@ def neumann_entropy(tensor):
         e = alg.eigvals(tensor)
         # temp_abs = torch.abs(e)
         temp_abs = e.real
-        temp = torch.log(temp_abs)
+        temp = torch.log(temp_abs).real
-        temp[temp == float("Inf")] = 0
+        temp = torch.nan_to_num(temp,
-        temp[temp == float("-Inf")] = 0
+                                nan=0.0, posinf=0.0, neginf=0.0)
         return -1 * torch.sum(temp_abs * temp)
     elif len(tensor_size) > 2:
         for i, x in enumerate(tensor):
@@ -59,9 +59,9 @@ def neumann_entropy(tensor):
                 e = alg.eigvals(t)
                 # temp_abs = torch.abs(e)
                 temp_abs = e.real
-                temp = torch.log(temp_abs)
+                temp = torch.log(temp_abs).real
-                temp[temp == float("Inf")] = 0
+                temp = torch.nan_to_num(temp,
-                temp[temp == float("-Inf")] = 0
+                                        nan=0.0, posinf=0.0, neginf=0.0)
                 return -1 * torch.sum(temp_abs * temp)
 
 

general_plots.py

@@ -3,8 +3,9 @@ import functions as aux
 
 eff_df = aux.load_pickle("efficiency_data.pkl")
 
-bayes_cifar_entropy = aux.load_pickle("bayes_data_cifar_ne.pkl")
+# bayes_cifar_entropy = aux.load_pickle("bayes_data_cifar_ne.pkl")
-bayes_mnist_entropy = aux.load_pickle("bayes_data_mnist_ne.pkl")
+# lenet_mnist_entropy = aux.load_pickle("lenet_data_mnist_ne.pkl")
+entropy_data = aux.load_pickle("entropy_data.pkl")
 
 bayes_keys = ['conv1.W_mu', 'conv1.W_rho', 'conv1.bias_mu', 'conv1.bias_rho',
               'conv2.W_mu', 'conv2.W_rho', 'conv2.bias_mu', 'conv2.bias_rho',
@@ -16,17 +17,13 @@ lenet_keys = ['conv1.weight', 'conv1.bias', 'conv2.weight', 'conv2.bias',
               'fc1.weight', 'fc1.bias', 'fc2.weight', 'fc2.bias', 'fc3.weight',
               'fc3.bias']
 
-"""
 for size in range(1, 8):
-    if size != 3:
+    if size != 8:
-        plt.plot(eff_df['CIFAR']['BCNN'][size], label='Size {}'.format(size))
+        plt.plot(eff_df['CIFAR']['BCNN'][size],
+                 label='Efficiency size {}'.format(size))
+        plt.plot(entropy_data['CIFAR']['BCNN'][size],
+                 label='Entropy size {}'.format(size))
 
-plt.legend(loc='upper right')
+# plt.legend(loc='upper right')
-plt.show()
+plt.legend(loc='lower right')
-"""
-temp = []
-for epoch in range(0, 100):
-    temp.append(bayes_cifar_entropy[1][epoch]['conv2.W_mu'])
-
-plt.plot(temp)
 plt.show()

get_entropy.py

@@ -1,10 +1,30 @@
 import functions as aux
+import statistics as st
 
 models_bayes_cifar = aux.load_pickle("bayes_data_cifar.pkl")
 models_bayes_mnist = aux.load_pickle("bayes_data_mnist.pkl")
 models_lenet_cifar = aux.load_pickle("lenet_data_cifar.pkl")
 models_lenet_mnist = aux.load_pickle("lenet_data_mnist.pkl")
 
+entropy_data = {'CIFAR':
+                {'BCNN':
+                 {1: None, 2: None, 3: None, 4: None,
+                  5: None, 6: None, 7: None},
+                 'LeNet':
+                 {1: None, 2: None, 3: None, 4: None,
+                  5: None, 6: None, 7: None}
+                 },
+                'MNIST':
+                {'BCNN':
+                 {1: None, 2: None, 3: None, 4: None,
+                  5: None, 6: None, 7: None},
+                 'LeNet':
+                 {1: None, 2: None, 3: None, 4: None,
+                  5: None, 6: None, 7: None}
+                 },
+                }
+
+"""
 bayes_keys = ['conv1.W_mu', 'conv1.W_rho', 'conv1.bias_mu', 'conv1.bias_rho',
               'conv2.W_mu', 'conv2.W_rho', 'conv2.bias_mu', 'conv2.bias_rho',
               'fc1.W_mu', 'fc1.W_rho', 'fc1.bias_mu', 'fc1.bias_rho',
@@ -14,6 +34,16 @@ bayes_keys = ['conv1.W_mu', 'conv1.W_rho', 'conv1.bias_mu', 'conv1.bias_rho',
 lenet_keys = ['conv1.weight', 'conv1.bias', 'conv2.weight', 'conv2.bias',
               'fc1.weight', 'fc1.bias', 'fc2.weight', 'fc2.bias', 'fc3.weight',
               'fc3.bias']
+"""
+
+bayes_keys = ['conv1.W_mu', 'conv1.W_rho',
+              'conv2.W_mu', 'conv2.W_rho',
+              'fc1.W_mu', 'fc1.W_rho',
+              'fc2.W_mu', 'fc2.W_rho',
+              'fc3.W_mu', 'fc3.W_rho']
+
+lenet_keys = ['conv1.weight', 'conv2.weight',
+              'fc1.weight', 'fc2.weight', 'fc3.weight']
 
 for model_size in range(1, 8):
     for epoch in range(0, 100):
@@ -25,7 +55,21 @@ for model_size in range(1, 8):
                         )
                     )
 
-aux.save_pickle("bayes_data_cifar_ne.pkl", models_bayes_cifar)
+for size in range(1, 8):
+    temp_epoch = []
+    for epoch in range(0, 100):
+        temp_mean = []
+        for layer in bayes_keys:
+            temp_mean.append(
+                models_bayes_cifar[size][epoch][layer].item()
+                )
+        temp_mean = st.mean(temp_mean)
+        temp_epoch.append(
+            temp_mean
+        )
+    entropy_data['CIFAR']['BCNN'][size] = temp_epoch
+
+# aux.save_pickle("bayes_data_cifar_ne.pkl", models_bayes_cifar)
 del models_bayes_cifar
 
 for model_size in range(1, 8):
@@ -38,7 +82,21 @@ for model_size in range(1, 8):
                             )
                     )
 
-aux.save_pickle("bayes_data_mnist_ne.pkl", models_bayes_mnist)
+for size in range(1, 8):
+    temp_epoch = []
+    for epoch in range(0, 100):
+        temp_mean = []
+        for layer in bayes_keys:
+            temp_mean.append(
+                models_bayes_mnist[size][epoch][layer].item()
+                )
+        temp_mean = st.mean(temp_mean)
+        temp_epoch.append(
+            temp_mean
+        )
+    entropy_data['MNIST']['BCNN'][size] = temp_epoch
+
+# aux.save_pickle("bayes_data_mnist_ne.pkl", models_bayes_mnist)
 del models_bayes_mnist
 
 for model_size in range(1, 8):
@@ -51,7 +109,21 @@ for model_size in range(1, 8):
                         )
                     )
 
-aux.save_pickle("lenet_data_cifar_ne.pkl", models_lenet_cifar)
+for size in range(1, 8):
+    temp_epoch = []
+    for epoch in range(0, 100):
+        temp_mean = []
+        for layer in lenet_keys:
+            temp_mean.append(
+                models_lenet_cifar[size][epoch][layer].item()
+                )
+        temp_mean = st.mean(temp_mean)
+        temp_epoch.append(
+            temp_mean
+        )
+    entropy_data['CIFAR']['LeNet'][size] = temp_epoch
+
+# aux.save_pickle("lenet_data_cifar_ne.pkl", models_lenet_cifar)
 del models_lenet_cifar
 
 for model_size in range(1, 8):
@@ -64,5 +136,22 @@ for model_size in range(1, 8):
                         )
                     )
 
-aux.save_pickle("lenet_data_mnist_ne.pkl", models_lenet_mnist)
+for size in range(1, 8):
+    temp_epoch = []
+    for epoch in range(0, 100):
+        temp_mean = []
+        for layer in lenet_keys:
+            temp_mean.append(
+                models_lenet_mnist[size][epoch][layer].item()
+                )
+        temp_mean = st.mean(temp_mean)
+        temp_epoch.append(
+            temp_mean
+        )
+    entropy_data['MNIST']['LeNet'][size] = temp_epoch
+
+
+# aux.save_pickle("lenet_data_mnist_ne.pkl", models_lenet_mnist)
 del models_lenet_mnist
+
+aux.save_pickle("entropy_data.pkl", entropy_data)