From b070fda52294d282bafecf1077c9efae8d2ac79f Mon Sep 17 00:00:00 2001
From: Sun Jin Kim <sk2521@ic.ac.uk>
Date: Mon, 25 Apr 2022 18:17:53 +0100
Subject: [PATCH] remove toy_flag parameter from aa_learners
---
.../autoaugment_learners/aa_learner.py | 92 ++++++++-----------
.../autoaugment_learners/evo_learner.py | 31 ++++---
.../autoaugment_learners/gru_learner.py | 5 +-
.../randomsearch_learner.py | 5 +-
.../autoaugment_learners/ucb_learner.py | 4 +-
MetaAugment/main.py | 6 ++
benchmark/scripts/04_22_ci_gru.py | 2 -
benchmark/scripts/04_22_ci_rs.py | 2 -
benchmark/scripts/04_22_fm_gru.py | 2 -
benchmark/scripts/04_22_fm_rs.py | 2 -
docs/source/usage/tutorial_for_team.rst | 1 -
temp_util/wapp_util.py | 1 -
test/MetaAugment/test_aa_learner.py | 1 -
test/MetaAugment/test_gru_learner.py | 1 -
test/MetaAugment/test_randomsearch_learner.py | 1 -
test/MetaAugment/test_ucb_learner.py | 1 -
16 files changed, 65 insertions(+), 92 deletions(-)
diff --git a/MetaAugment/autoaugment_learners/aa_learner.py b/MetaAugment/autoaugment_learners/aa_learner.py
index e4460cbf..44a86aa3 100644
--- a/MetaAugment/autoaugment_learners/aa_learner.py
+++ b/MetaAugment/autoaugment_learners/aa_learner.py
@@ -51,8 +51,7 @@ class aa_learner:
discrete_p_m=False,
# hyperparameters for when training the child_network
batch_size=32,
- toy_flag=False,
- toy_size=0.1,
+ toy_size=1,
learning_rate=1e-1,
max_epochs=float('inf'),
early_stop_num=20,
@@ -74,7 +73,6 @@ class aa_learner:
algorithm, etc.). Defaults to False
batch_size (int, optional): child_network training parameter. Defaults to 32.
- toy_flag (bool, optional): child_network training parameter. Defaults to False.
toy_size (int, optional): child_network training parameter. ratio of original
dataset used in toy dataset. Defaults to 0.1.
learning_rate (float, optional): child_network training parameter. Defaults to 1e-2.
@@ -90,7 +88,6 @@ class aa_learner:
# related to training of the child_network
self.batch_size = batch_size
- self.toy_flag = toy_flag
self.toy_size = toy_size
self.learning_rate = learning_rate
@@ -299,7 +296,7 @@ class aa_learner:
reward = self.test_autoaugment_policy(policy,
child_network_architecture,
train_dataset,
- test_dataset, toy_flag)
+ test_dataset)
self.history.append((policy, reward))
"""
@@ -326,8 +323,6 @@ class aa_learner:
of it.
train_dataset (torchvision.dataset.vision.VisionDataset)
test_dataset (torchvision.dataset.vision.VisionDataset)
- toy_flag (boolean): Whether we want to obtain a toy version of
- train_dataset and test_dataset and use those.
logging (boolean): Whether we want to save logs
Returns:
@@ -361,17 +356,11 @@ class aa_learner:
train_dataset.transform = train_transform
# create Dataloader objects out of the Dataset objects
- if self.toy_flag:
- train_loader, test_loader = create_toy(train_dataset,
- test_dataset,
- batch_size=self.batch_size,
- n_samples=self.toy_size,
- seed=100)
- else:
- train_loader = torch.utils.data.DataLoader(train_dataset,
- batch_size=self.batch_size)
- test_loader = torch.utils.data.DataLoader(test_dataset,
- batch_size=self.batch_size)
+ train_loader, test_loader = create_toy(train_dataset,
+ test_dataset,
+ batch_size=self.batch_size,
+ n_samples=self.toy_size,
+ seed=100)
# train the child network with the dataloaders equipped with our specific policy
accuracy = train_child_network(child_network,
@@ -392,39 +381,38 @@ class aa_learner:
return accuracy
- def demo_plot(self, train_dataset, test_dataset, child_network_architecture, n=5):
- """
- I made this to plot a couple of accuracy graphs to help manually tune my gradient
- optimizer hyperparameters.
+ # def demo_plot(self, train_dataset, test_dataset, child_network_architecture, n=5):
+ # """
+ # I made this to plot a couple of accuracy graphs to help manually tune my gradient
+ # optimizer hyperparameters.
- Saves a plot of `n` training accuracy graphs overlapped.
- """
+ # Saves a plot of `n` training accuracy graphs overlapped.
+ # """
- acc_lists = []
-
- # This is dummy code
- # test out `n` random policies
- for _ in range(n):
- policy = self.generate_new_policy()
-
- pprint(policy)
- reward, acc_list = self.test_autoaugment_policy(policy,
- child_network_architecture,
- train_dataset,
- test_dataset,
- toy_flag=self.toy_flag,
- logging=True)
-
- self.history.append((policy, reward))
- acc_lists.append(acc_list)
-
- for acc_list in acc_lists:
- plt.plot(acc_list)
- plt.title('I ran 5 random policies to see if there is any sign of \
- catastrophic failure during training. If there are \
- any lines which reach significantly lower (>10%) \
- accuracies, you might want to tune the hyperparameters')
- plt.xlabel('epoch')
- plt.ylabel('accuracy')
- plt.show()
- plt.savefig('training_graphs_without_policies')
\ No newline at end of file
+ # acc_lists = []
+
+ # # This is dummy code
+ # # test out `n` random policies
+ # for _ in range(n):
+ # policy = self.generate_new_policy()
+
+ # pprint(policy)
+ # reward, acc_list = self.test_autoaugment_policy(policy,
+ # child_network_architecture,
+ # train_dataset,
+ # test_dataset,
+ # logging=True)
+
+ # self.history.append((policy, reward))
+ # acc_lists.append(acc_list)
+
+ # for acc_list in acc_lists:
+ # plt.plot(acc_list)
+ # plt.title('I ran 5 random policies to see if there is any sign of \
+ # catastrophic failure during training. If there are \
+ # any lines which reach significantly lower (>10%) \
+ # accuracies, you might want to tune the hyperparameters')
+ # plt.xlabel('epoch')
+ # plt.ylabel('accuracy')
+ # plt.show()
+ # plt.savefig('training_graphs_without_policies')
\ No newline at end of file
diff --git a/MetaAugment/autoaugment_learners/evo_learner.py b/MetaAugment/autoaugment_learners/evo_learner.py
index e9a65865..682061ef 100644
--- a/MetaAugment/autoaugment_learners/evo_learner.py
+++ b/MetaAugment/autoaugment_learners/evo_learner.py
@@ -14,28 +14,29 @@ import MetaAugment.child_networks as cn
class evo_learner(aa_learner):
def __init__(self,
- sp_num=1,
- num_solutions = 5,
- num_parents_mating = 3,
- learning_rate = 1e-1,
+ # search space settings
+ discrete_p_m=False,
+ exclude_method=[],
+ sp_num=5,
+ p_bins=10,
+ m_bins=10,
+ # child network settings
+ learning_rate=1e-1,
max_epochs=float('inf'),
early_stop_num=20,
- p_bins = 1,
- m_bins = 1,
- discrete_p_m=False,
batch_size=8,
- toy_flag=False,
- toy_size=0.1,
- exclude_method=[],
- controller = None
+ toy_size=1,
+ # evolutionary learner specific settings
+ num_solutions=5,
+ num_parents_mating=3,
+ controller=None
):
- super().__init__(sp_num,
- p_bins,
- m_bins,
+ super().__init__(sp_num=sp_num,
+ p_bins=p_bins,
+ m_bins=m_bins,
discrete_p_m=discrete_p_m,
batch_size=batch_size,
- toy_flag=toy_flag,
toy_size=toy_size,
learning_rate=learning_rate,
max_epochs=max_epochs,
diff --git a/MetaAugment/autoaugment_learners/gru_learner.py b/MetaAugment/autoaugment_learners/gru_learner.py
index db8205d5..6955257f 100644
--- a/MetaAugment/autoaugment_learners/gru_learner.py
+++ b/MetaAugment/autoaugment_learners/gru_learner.py
@@ -52,8 +52,7 @@ class gru_learner(aa_learner):
discrete_p_m=False,
# hyperparameters for when training the child_network
batch_size=8,
- toy_flag=False,
- toy_size=0.1,
+ toy_size=1,
learning_rate=1e-1,
max_epochs=float('inf'),
early_stop_num=20,
@@ -83,7 +82,6 @@ class gru_learner(aa_learner):
m_bins=m_bins,
discrete_p_m=True,
batch_size=batch_size,
- toy_flag=toy_flag,
toy_size=toy_size,
learning_rate=learning_rate,
max_epochs=max_epochs,
@@ -244,7 +242,6 @@ if __name__=='__main__':
agent = gru_learner(
sp_num=7,
- toy_flag=True,
toy_size=0.01,
batch_size=32,
learning_rate=0.1,
diff --git a/MetaAugment/autoaugment_learners/randomsearch_learner.py b/MetaAugment/autoaugment_learners/randomsearch_learner.py
index 09f6626f..71d8bc1a 100644
--- a/MetaAugment/autoaugment_learners/randomsearch_learner.py
+++ b/MetaAugment/autoaugment_learners/randomsearch_learner.py
@@ -43,8 +43,7 @@ class randomsearch_learner(aa_learner):
discrete_p_m=True,
# hyperparameters for when training the child_network
batch_size=8,
- toy_flag=False,
- toy_size=0.1,
+ toy_size=1,
learning_rate=1e-1,
max_epochs=float('inf'),
early_stop_num=30,
@@ -55,7 +54,6 @@ class randomsearch_learner(aa_learner):
m_bins=m_bins,
discrete_p_m=discrete_p_m,
batch_size=batch_size,
- toy_flag=toy_flag,
toy_size=toy_size,
learning_rate=learning_rate,
max_epochs=max_epochs,
@@ -185,7 +183,6 @@ if __name__=='__main__':
agent = randomsearch_learner(
sp_num=7,
- toy_flag=True,
toy_size=0.01,
batch_size=4,
learning_rate=0.05,
diff --git a/MetaAugment/autoaugment_learners/ucb_learner.py b/MetaAugment/autoaugment_learners/ucb_learner.py
index dc82c2ee..aa6cd685 100644
--- a/MetaAugment/autoaugment_learners/ucb_learner.py
+++ b/MetaAugment/autoaugment_learners/ucb_learner.py
@@ -25,8 +25,7 @@ class ucb_learner(randomsearch_learner):
discrete_p_m=True,
# hyperparameters for when training the child_network
batch_size=8,
- toy_flag=False,
- toy_size=0.1,
+ toy_size=1,
learning_rate=1e-1,
max_epochs=float('inf'),
early_stop_num=30,
@@ -39,7 +38,6 @@ class ucb_learner(randomsearch_learner):
m_bins=m_bins,
discrete_p_m=discrete_p_m,
batch_size=batch_size,
- toy_flag=toy_flag,
toy_size=toy_size,
learning_rate=learning_rate,
max_epochs=max_epochs,
diff --git a/MetaAugment/main.py b/MetaAugment/main.py
index af1f311d..51c9070a 100644
--- a/MetaAugment/main.py
+++ b/MetaAugment/main.py
@@ -11,6 +11,12 @@ import torchvision.datasets as datasets
def create_toy(train_dataset, test_dataset, batch_size, n_samples, seed=100):
+ if n_samples==1:
+ # push into DataLoader
+ train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size)
+ test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size)
+ return train_loader, test_loader
+
# shuffle and take first n_samples %age of training dataset
shuffle_order_train = np.random.RandomState(seed=seed).permutation(len(train_dataset))
shuffled_train_dataset = torch.utils.data.Subset(train_dataset, shuffle_order_train)
diff --git a/benchmark/scripts/04_22_ci_gru.py b/benchmark/scripts/04_22_ci_gru.py
index 194a5f23..155b7c92 100644
--- a/benchmark/scripts/04_22_ci_gru.py
+++ b/benchmark/scripts/04_22_ci_gru.py
@@ -12,8 +12,6 @@ from .util_04_22 import *
config = {
'sp_num' : 3,
'learning_rate' : 1e-1,
- 'toy_flag' : False,
-# 'toy_flag' : True,
# 'toy_size' : 0.001,
'batch_size' : 32,
'max_epochs' : 100,
diff --git a/benchmark/scripts/04_22_ci_rs.py b/benchmark/scripts/04_22_ci_rs.py
index c6dd5f4d..e1279b1e 100644
--- a/benchmark/scripts/04_22_ci_rs.py
+++ b/benchmark/scripts/04_22_ci_rs.py
@@ -12,8 +12,6 @@ from .util_04_22 import *
config = {
'sp_num' : 3,
'learning_rate' : 1e-1,
- 'toy_flag' : False,
-# 'toy_flag' : True,
# 'toy_size' : 0.001,
'batch_size' : 32,
'max_epochs' : 100,
diff --git a/benchmark/scripts/04_22_fm_gru.py b/benchmark/scripts/04_22_fm_gru.py
index 799e439e..807d0177 100644
--- a/benchmark/scripts/04_22_fm_gru.py
+++ b/benchmark/scripts/04_22_fm_gru.py
@@ -12,8 +12,6 @@ from .util_04_22 import *
config = {
'sp_num' : 3,
'learning_rate' : 1e-1,
- 'toy_flag' : False,
-# 'toy_flag' : True,
# 'toy_size' : 0.001,
'batch_size' : 32,
'max_epochs' : 100,
diff --git a/benchmark/scripts/04_22_fm_rs.py b/benchmark/scripts/04_22_fm_rs.py
index 6b983284..dfe71958 100644
--- a/benchmark/scripts/04_22_fm_rs.py
+++ b/benchmark/scripts/04_22_fm_rs.py
@@ -12,8 +12,6 @@ from .util_04_22 import *
config = {
'sp_num' : 3,
'learning_rate' : 1e-1,
- 'toy_flag' : False,
-# 'toy_flag' : True,
# 'toy_size' : 0.001,
'batch_size' : 32,
'max_epochs' : 100,
diff --git a/docs/source/usage/tutorial_for_team.rst b/docs/source/usage/tutorial_for_team.rst
index d4cebf46..1c81cd7c 100644
--- a/docs/source/usage/tutorial_for_team.rst
+++ b/docs/source/usage/tutorial_for_team.rst
@@ -57,7 +57,6 @@ can use any other learner in place of random search learner as well)
# aa_agent = aal.ac_learner()
aa_agent = aal.randomsearch_learner(
sp_num=7,
- toy_flag=True,
toy_size=0.01,
batch_size=4,
learning_rate=0.05,
diff --git a/temp_util/wapp_util.py b/temp_util/wapp_util.py
index e48d1c31..5ea55018 100644
--- a/temp_util/wapp_util.py
+++ b/temp_util/wapp_util.py
@@ -125,7 +125,6 @@ def parse_users_learner_spec(
# use an aa_learner. in this case, a rs learner
agent = aal.randomsearch_learner(batch_size=batch_size,
- toy_flag=True,
learning_rate=learning_rate,
toy_size=toy_size,
max_epochs=max_epochs,
diff --git a/test/MetaAugment/test_aa_learner.py b/test/MetaAugment/test_aa_learner.py
index 29af4f6d..b1524988 100644
--- a/test/MetaAugment/test_aa_learner.py
+++ b/test/MetaAugment/test_aa_learner.py
@@ -82,7 +82,6 @@ def test_test_autoaugment_policy():
p_bins=11,
m_bins=10,
discrete_p_m=True,
- toy_flag=True,
toy_size=0.004,
max_epochs=20,
early_stop_num=10
diff --git a/test/MetaAugment/test_gru_learner.py b/test/MetaAugment/test_gru_learner.py
index b5c695cf..cd52b0e9 100644
--- a/test/MetaAugment/test_gru_learner.py
+++ b/test/MetaAugment/test_gru_learner.py
@@ -42,7 +42,6 @@ def test_learn():
agent = aal.gru_learner(
sp_num=7,
- toy_flag=True,
toy_size=0.001,
batch_size=32,
learning_rate=0.05,
diff --git a/test/MetaAugment/test_randomsearch_learner.py b/test/MetaAugment/test_randomsearch_learner.py
index 29cd812b..61e9f9cd 100644
--- a/test/MetaAugment/test_randomsearch_learner.py
+++ b/test/MetaAugment/test_randomsearch_learner.py
@@ -51,7 +51,6 @@ def test_learn():
agent = aal.randomsearch_learner(
sp_num=7,
- toy_flag=True,
toy_size=0.001,
batch_size=32,
learning_rate=0.05,
diff --git a/test/MetaAugment/test_ucb_learner.py b/test/MetaAugment/test_ucb_learner.py
index 7c6635ff..3f37f3e5 100644
--- a/test/MetaAugment/test_ucb_learner.py
+++ b/test/MetaAugment/test_ucb_learner.py
@@ -21,7 +21,6 @@ def test_ucb_learner():
discrete_p_m=True,
# hyperparameters for when training the child_network
batch_size=8,
- toy_flag=True,
toy_size=0.001,
learning_rate=1e-1,
max_epochs=float('inf'),
--
GitLab