84 lines
2.1 KiB
Python
Executable File
84 lines
2.1 KiB
Python
Executable File
import pickle
|
|
from time import sleep
|
|
|
|
from gpu_power_func import total_watt_consumed
|
|
|
|
with open("configuration.pkl", "rb") as file:
|
|
while True:
|
|
try:
|
|
cfg = pickle.load(file)
|
|
except EOFError:
|
|
break
|
|
|
|
|
|
def non_decreasing(L):
|
|
return all(x <= y for x, y in zip(L, L[1:]))
|
|
|
|
|
|
def non_increasing(L):
|
|
return all(x >= y for x, y in zip(L, L[1:]))
|
|
|
|
|
|
def monotonic(L):
|
|
return non_decreasing(L) or non_increasing(L)
|
|
|
|
|
|
def strictly_increasing(L):
|
|
return all(x < y for x, y in zip(L, L[1:]))
|
|
|
|
|
|
def strictly_decreasing(L):
|
|
return all(x > y for x, y in zip(L, L[1:]))
|
|
|
|
|
|
def strictly_monotonic(L):
|
|
return strictly_increasing(L) or strictly_decreasing(L)
|
|
|
|
|
|
def e_stop(
|
|
early_stopping: list,
|
|
train_acc: float,
|
|
epoch: int,
|
|
patience: int = 4,
|
|
sensitivity: float = 1e-9,
|
|
):
|
|
early_stopping.append(train_acc)
|
|
if patience in (0, 1):
|
|
print("Stopping Early")
|
|
return 1
|
|
if epoch % patience == 0 and epoch > 0:
|
|
early_stopping = early_stopping[-patience : len(early_stopping)]
|
|
ini = early_stopping.pop(0)
|
|
early_stopping = list(map(lambda x: x - sensitivity, early_stopping))
|
|
early_stopping.insert(0, ini)
|
|
values = ""
|
|
for i, v in enumerate(early_stopping):
|
|
values += f"Value {i+1}: {v} > "
|
|
print(values)
|
|
if (train_acc > 0.5) and not strictly_increasing(early_stopping):
|
|
print("Stopping Early")
|
|
return 1
|
|
del early_stopping[:]
|
|
return 0
|
|
|
|
|
|
def energy_bound(threshold: float = 100000.0):
|
|
"""Stops training when a specified amount of energy is consumed"""
|
|
try:
|
|
energy = total_watt_consumed(cfg["pickle_path"])
|
|
except Exception as e:
|
|
sleep(3)
|
|
energy = total_watt_consumed(cfg["pickle_path"])
|
|
print("Energy used: {}".format(energy))
|
|
if energy > threshold:
|
|
print("Energy bound achieved")
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def accuracy_bound(train_acc: float, threshold: float = 0.99):
|
|
if train_acc >= threshold:
|
|
print("Accuracy bound achieved")
|
|
return 1
|
|
return 0
|