diff --git a/04_22_evo.py b/04_22_evo.py
new file mode 100644
index 0000000000000000000000000000000000000000..6b84237c38c4f1af7a4853bfcc9acc956ea0dd49
--- /dev/null
+++ b/04_22_evo.py
@@ -0,0 +1,209 @@
+import torchvision.datasets as datasets
+import torchvision
+import torch
+
+import autoaug.child_networks as cn
+import autoaug.autoaugment_learners as aal
+
+
+controller = cn.EasyNet(img_height=32, img_width=32, num_labels=16*10, img_channels=3)
+
+config = {
+ 'sp_num' : 5,
+ 'learning_rate' : 1e-1,
+ 'batch_size' : 32,
+ 'max_epochs' : 100,
+ 'early_stop_num' : 10,
+ 'controller' : controller,
+ 'num_solutions' : 10,
+ }
+
+
+
+
+import torch
+
+import autoaug.autoaugment_learners as aal
+
+import pprint
+
+"""
+testing GruLearner and RsLearner on
+
+ fashionmnist with simple net
+
+ and
+
+ cifar10 with lenet
+
+"""
+if torch.cuda.is_available():
+ device = torch.device('cuda')
+else:
+ device = torch.device('cpu')
+
+
+
+
+
+
+def run_benchmark(
+ save_file,
+ train_dataset,
+ test_dataset,
+ child_network_architecture,
+ agent_arch,
+ config,
+ total_iter=150,
+ ):
+ try:
+ # try to load agent
+ with open(save_file, 'rb') as f:
+ agent = torch.load(f, map_location=device)
+ except FileNotFoundError:
+ # if agent hasn't been saved yet, initialize the agent
+ agent = agent_arch(**config)
+
+
+ # if history is not length total_iter yet(if total_iter
+ # different policies haven't been tested yet), keep running
+
+ print("agent history: ", agent.history)
+ while len(agent.history)<total_iter:
+ print(f'{len(agent.history)} / {total_iter}')
+ # run 1 iteration (test one new policy and update the GRU)
+ agent.learn(
+ train_dataset=train_dataset,
+ test_dataset=test_dataset,
+ child_network_architecture=child_network_architecture,
+ iterations=1
+ )
+ # save agent every iteration
+ with open(save_file, 'wb+') as f:
+ torch.save(agent, f)
+
+ print('run_benchmark closing')
+
+
+def get_mega_policy(history, n):
+ """
+ we get the best n policies from an agent's history,
+ concatenate them to form our best mega policy
+
+ Args:
+ history (list[tuple])
+ n (int)
+
+ Returns:
+ list[float]: validation accuracies
+ """
+ assert len(history) >= n
+
+ # agent.history is a list of (policy(list), val_accuracy(float)) tuples
+ sorted_history = sorted(history, key=lambda x:x[1], reverse=True) # sort wrt acc
+
+ best_history = sorted_history[:n]
+
+ megapolicy = []
+ # we also want to keep track of how good the best policies were
+ # maybe if we add them all up, they'll become worse! Hopefully better tho
+ orig_accs = []
+
+ for policy,acc in best_history:
+ for subpolicy in policy:
+ megapolicy.append(subpolicy)
+ orig_accs.append(acc)
+
+ return megapolicy, orig_accs
+
+
+def rerun_best_policy(
+ agent_pickle,
+ accs_txt,
+ train_dataset,
+ test_dataset,
+ child_network_architecture,
+ config,
+ repeat_num
+ ):
+
+ with open(agent_pickle, 'rb') as f:
+ agent = torch.load(f)
+
+ megapol, orig_accs = get_mega_policy(agent.history,3)
+ print('mega policy to be tested:')
+ pprint.pprint(megapol)
+ print(orig_accs)
+
+ accs=[]
+ for _ in range(repeat_num):
+ print(f'{_}/{repeat_num}')
+ temp_agent = aal.AaLearner(**config)
+ accs.append(
+ temp_agent._test_autoaugment_policy(megapol,
+ child_network_architecture,
+ train_dataset,
+ test_dataset,
+ logging=False)
+ )
+ with open(accs_txt, 'w') as f:
+ f.write(pprint.pformat(megapol))
+ f.write(str(accs))
+ f.write(f'original small policys accuracies: {orig_accs}')
+
+
+
+
+# # CIFAR10 with LeNet
+train_dataset = datasets.CIFAR10(root='./datasets/cifar10/train',
+ train=True, download=True, transform=None)
+test_dataset = datasets.CIFAR10(root='./datasets/cifar10/train',
+ train=False, download=True,
+ transform=torchvision.transforms.ToTensor())
+child_network_architecture = cn.LeNet(
+ img_height=32,
+ img_width=32,
+ num_labels=10,
+ img_channels=3
+ )
+
+# save_dir='./benchmark/pickles/04_22_cf_ln_rssad'
+
+# # evo
+# run_benchmark(
+# save_file=save_dir+'.pkl',
+# train_dataset=train_dataset,
+# test_dataset=test_dataset,
+# child_network_architecture=child_network_architecture,
+# agent_arch=aal.EvoLearner,
+# config=config,
+# )
+
+# # rerun_best_policy(
+# # agent_pickle=save_dir+'.pkl',
+# # accs_txt=save_dir+'.txt',
+# # train_dataset=train_dataset,
+# # test_dataset=test_dataset,
+# # child_network_architecture=child_network_architecture,
+# # config=config,
+# # repeat_num=5
+# # )
+
+
+
+megapol = [(('ShearY', 0.5, 5), ('Posterize', 0.6, 5)), (('Color', 1.0, 9), ('Contrast', 1.0, 9)), (('TranslateX', 0.5, 5), ('Posterize', 0.5, 5)), (('TranslateX', 0.5, 5), ('Posterize', 0.5, 5)), (('Color', 0.5, 5), ('Posterize', 0.5, 5))]
+
+
+accs=[]
+for _ in range(10):
+ print(f'{_}/{10}')
+ temp_agent = aal.evo_learner(**config)
+ accs.append(
+ temp_agent.test_autoaugment_policy(megapol,
+ child_network_architecture,
+ train_dataset,
+ test_dataset,
+ logging=False)
+ )
+
+print("CIPHAR10 accs: ", accs)
diff --git a/autoaug/autoaugment_learners/AaLearner.py b/autoaug/autoaugment_learners/AaLearner.py
index c0566d4edba29079f70e1d3e0334393894796a03..638b92237e463f035792c3157b08a9100d9fa02c 100644
--- a/autoaug/autoaugment_learners/AaLearner.py
+++ b/autoaug/autoaugment_learners/AaLearner.py
@@ -349,7 +349,7 @@ class AaLearner:
train_dataset,
test_dataset,
logging=False,
- print_every_epoch=False):
+ print_every_epoch=True):
"""
Given a policy (using AutoAugment paper terminology), we train a child network
using the policy and return the accuracy (how good the policy is for the dataset and
diff --git a/autoaug/autoaugment_learners/EvoLearner.py b/autoaug/autoaugment_learners/EvoLearner.py
index c78d0bb87e8951c52afba14855e5e1d7d6018b4a..c01aafa857a654072d129b8a724552e1adc1a7bc 100644
--- a/autoaug/autoaugment_learners/EvoLearner.py
+++ b/autoaug/autoaugment_learners/EvoLearner.py
@@ -10,28 +10,85 @@ import autoaug.controller_networks as cont_n
class EvoLearner(AaLearner):
- """evo algorithm
+ """Evolutionary Strategy learner
- long explanatino here
+ This learner generates neural networks that predict optimal augmentation
+ policies. Hence, there is no backpropagation or gradient descent. Instead,
+ training is done by randomly changing weights of the 'parent' networks, where
+ parents are determined by their ability to produce policies that
+ increase the accuracy of the child network.
Args:
- AaLearner (_type_): _description_
+ AaLearner:
+ Base class of all-auto augmentation learners.
-
+ sp_num: int, default 5
+ Number of subpolicies to keep in the final policy
+
+ p_bins: int, default 1
+ Number of probability bins for the controller network.
+
+ m_bins: int, default 1
+ Number of magnitude bins for the controller network
+
+ discrete_p_m: bool, default False
+ Boolean value to set if there are discrete or continuous
+ probability and mangitude bins (if False; p_bins, m_bins = 1)
+
+ exclude_method: list, default []
+ List of augmentations to be excluded from the search space
+
+ (Child Network Args)
+
+ learning_rate: float, default 1e-6
+ Learning rate of the child network
+
+ max_epochs: float, default float('inf')
+ Theoretical maximum number of epochs that the child network
+ can be trained on
+
+ early_stop_num: int, default 20
+ Criteria for early stopping. I.e. if the network has not improved
+ after early_stop_num iterations, the training is stopped
+
+ batch_size: int, default 8
+ Batch size for the datasets
+
+ toy_size: float, default 1
+ If a toy dataset is created, it will be of size toy_size compared
+ to the original dataset
+
+ (Evolutionary learner specific settings)
+
+ num_solutions: int, default 5
+ Number of offspring spawned at each generation of the algorithm
+
+ num_parents_mating: int, default 3
+ Number of networks chosen as parents for the next generation of networks
+
+ controller: Torch Network, default cont_n.EvoController
+ Controller network for the evolutionary algorithm
+
+
See Also
--------
Notes
-----
+ The Evolutionary algorithm runs in generations, and so batches of child networks
+ are trained at specific time intervals.
References
----------
+
Examples
--------
+ from autoaug.autoaugment_learners.EvlLearner import EvoLearner
+ evo_learner = EvoLearner()
"""
@@ -67,12 +124,13 @@ class EvoLearner(AaLearner):
)
# evolutionary algorithm settings
- self.controller = controller(
- fun_num=self.fun_num,
- p_bins=self.p_bins,
- m_bins=self.m_bins,
- sub_num_pol=self.sp_num
- )
+ # self.controller = controller(
+ # fun_num=self.fun_num,
+ # p_bins=self.p_bins,
+ # m_bins=self.m_bins,
+ # sub_num_pol=self.sp_num
+ # )
+ self.controller = controller
self.num_solutions = num_solutions
self.torch_ga = torchga.TorchGA(model=self.controller, num_solutions=num_solutions)
self.num_parents_mating = num_parents_mating
@@ -184,6 +242,7 @@ class EvoLearner(AaLearner):
Solution_idx -> Int
"""
+ print("learn0")
self.num_generations = iterations
self.history_best = []
@@ -244,6 +303,7 @@ class EvoLearner(AaLearner):
self.train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100)
count = 0
for idx, (test_x, label_x) in enumerate(self.train_loader):
+ print("here idx: ", idx)
count += 1
sub_pol = self._get_single_policy_cov(test_x)
@@ -254,8 +314,9 @@ class EvoLearner(AaLearner):
if idx == 0:
break
-
+ print("start test")
fit_val = self._test_autoaugment_policy(sub_pol,child_network_architecture,train_dataset,test_dataset)
+ print("end test")
self.running_policy.append((sub_pol, fit_val))
diff --git a/autoaug/autoaugment_learners/GenLearner.py b/autoaug/autoaugment_learners/GenLearner.py
index 349d71d3846da122360fff7579da96da950c9a87..b93d106b5a6aa72195a894965c5ae8ca47fc4507 100644
--- a/autoaug/autoaugment_learners/GenLearner.py
+++ b/autoaug/autoaugment_learners/GenLearner.py
@@ -195,6 +195,7 @@ class Genetic_learner(AaLearner):
policy = [self.gen_random_subpol()]
else:
policy = self.bin_to_subpol(random.choice(self.generate_children()))
+ print("Policy: ", policy)
reward = self._test_autoaugment_policy(policy,
child_network_architecture,
diff --git a/autoaug/main.py b/autoaug/main.py
index fd83f72d7e71e6a670335c8aafc9fc07c93c397b..5a2c0adca63d6327e3386cc9d1f040789105b931 100644
--- a/autoaug/main.py
+++ b/autoaug/main.py
@@ -50,7 +50,7 @@ def train_child_network(child_network,
early_stop_flag=True,
average_validation=[15,25],
logging=False,
- print_every_epoch=True):
+ print_every_epoch=False):
if torch.cuda.is_available():
device = torch.device('cuda')
else:
@@ -125,8 +125,8 @@ def train_child_network(child_network,
best_acc = total_val / (average_validation[1] - average_validation[0] + 1)
break
- if print_every_epoch:
- print('main.train_child_network best accuracy: ', best_acc)
+ # if print_every_epoch:
+ # print('main.train_child_network best accuracy: ', best_acc)
acc_log.append(acc)
_epoch+=1
diff --git a/extract_gen.py b/extract_gen.py
new file mode 100644
index 0000000000000000000000000000000000000000..9633cdf032d39c379da604d406bb514d88cb0fb9
--- /dev/null
+++ b/extract_gen.py
@@ -0,0 +1,12 @@
+import pickle
+
+with open('genetic_logs.pkl', 'rb') as f:
+ y = pickle.load(f)
+
+accs = [x[1] for x in y]
+
+print("accs: ", accs)
+print("len accs: ", len(accs))
+
+with open("genetic_accs.txt", "w") as output:
+ output.write(str(accs))
\ No newline at end of file
diff --git a/genetic_accs.txt b/genetic_accs.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5e3df47598d4275aa4d860e52ccbe589bf86493c
--- /dev/null
+++ b/genetic_accs.txt
@@ -0,0 +1 @@
+[0.6100000143051147, 0.7799999713897705, 0.6700000166893005, 0.1599999964237213, 0.7200000286102295, 0.7400000095367432, 0.550000011920929, 0.5600000023841858, 0.6000000238418579, 0.550000011920929, 0.699999988079071, 0.3700000047683716, 0.699999988079071, 0.7699999809265137, 0.6800000071525574, 0.38999998569488525, 0.7400000095367432, 0.5799999833106995, 0.3700000047683716, 0.4699999988079071, 0.7099999785423279, 0.6399999856948853, 0.6200000047683716, 0.6200000047683716, 0.6899999976158142, 0.7099999785423279, 0.1599999964237213, 0.49000000953674316, 0.7099999785423279, 0.3700000047683716, 0.699999988079071, 0.6299999952316284, 0.75, 0.6700000166893005, 0.6600000262260437, 0.7400000095367432, 0.6299999952316284, 0.5099999904632568, 0.5099999904632568, 0.1599999964237213, 0.6700000166893005, 0.38999998569488525, 0.6600000262260437, 0.5699999928474426, 0.5799999833106995, 0.3400000035762787, 0.5400000214576721, 0.6100000143051147, 0.09000000357627869, 0.75, 0.7900000214576721, 0.6800000071525574, 0.1599999964237213, 0.5099999904632568, 0.6800000071525574, 0.5, 0.6299999952316284, 0.7300000190734863, 0.6399999856948853, 0.7400000095367432, 0.5799999833106995, 0.7699999809265137, 0.6399999856948853, 0.7099999785423279, 0.49000000953674316, 0.30000001192092896, 0.7200000286102295, 0.46000000834465027, 0.6200000047683716, 0.4399999976158142, 0.1599999964237213, 0.23000000417232513, 0.6899999976158142, 0.1599999964237213, 0.7300000190734863, 0.7099999785423279, 0.800000011920929, 0.46000000834465027, 0.7099999785423279, 0.7300000190734863, 0.2199999988079071, 0.6700000166893005, 0.7099999785423279, 0.4000000059604645, 0.5400000214576721, 0.4300000071525574, 0.75, 0.1599999964237213, 0.6399999856948853, 0.6399999856948853, 0.6499999761581421, 0.7900000214576721, 0.7400000095367432, 0.6399999856948853, 0.5899999737739563, 0.7099999785423279, 0.5199999809265137, 0.5, 0.7099999785423279, 0.6299999952316284]
\ No newline at end of file
diff --git a/genetic_logs.pkl b/genetic_logs.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..3b78e572c81572887c4eb3cbdc19d19870637040
Binary files /dev/null and b/genetic_logs.pkl differ
diff --git a/ttest_gen.py b/ttest_gen.py
index 3dd462dac9e2f544498a218537e80fc2a939990d..26f7ffa3a067ff8fdeb359403bdf1603ebf1f2ba 100644
--- a/ttest_gen.py
+++ b/ttest_gen.py
@@ -9,19 +9,35 @@ from autoaug.autoaugment_learners.AaLearner import AaLearner
from autoaug.autoaugment_learners.GenLearner import Genetic_learner
import random
+import pickle
# train_dataset = datasets.MNIST(root='./datasets/mnist/train',
# train=True, download=True, transform=None)
# test_dataset = datasets.MNIST(root='./datasets/mnist/test',
# train=False, download=True, transform=torchvision.transforms.ToTensor())
-train_dataset = datasets.FashionMNIST(root='./datasets/fashionmnist/train',
+# train_dataset = datasets.FashionMNIST(root='./datasets/fashionmnist/train',
+# train=True, download=True, transform=None)
+# test_dataset = datasets.FashionMNIST(root='./datasets/fashionmnist/test',
+# train=False, download=True,
+# transform=torchvision.transforms.ToTensor())
+
+
+train_dataset = datasets.CIFAR10(root='./datasets/cifar10/train',
train=True, download=True, transform=None)
-test_dataset = datasets.FashionMNIST(root='./datasets/fashionmnist/test',
- train=False, download=True,
+test_dataset = datasets.CIFAR10(root='./datasets/cifar10/train',
+ train=False, download=True,
transform=torchvision.transforms.ToTensor())
+
child_network_architecture = cn.lenet
# child_network_architecture = cn.lenet()
+child_network_architecture = cn.LeNet(
+ img_height=32,
+ img_width=32,
+ num_labels=10,
+ img_channels=3
+ )
+
agent = Genetic_learner(
sp_num=2,
toy_size=0.01,
@@ -38,6 +54,12 @@ agent.learn(train_dataset,
child_network_architecture=child_network_architecture,
iterations=100)
-# with open('genetic_logs.pkl', 'wb') as file:
-# pickle.dump(agent.history, file)
-print(sorted(agent.history, key = lambda x: x[1]))
\ No newline at end of file
+with open('genetic_logs.pkl', 'wb') as file:
+ pickle.dump(agent.history, file)
+print(sorted(agent.history, key = lambda x: x[1], reverse = True))
+
+print("ACCURACIES IN TIME: ")
+
+for iter, (pol, acc) in enumerate(agent.history):
+ print("pol: ", pol)
+ print("acc: ", acc)
\ No newline at end of file