pytorch 学习率调整策略

lambda1 = lambda epoch: self.step // 10
scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=lambda1)

lambda1 = lambda epoch: math.sqrt(epoch)
scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=lambda1)

# 每经过25步，学习率乘以0.1
scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size=25, gamma=0.1)

# 当步数到达50， 75时，学习率乘以0.1
scheduler = torch.optim.lr_scheduler.MultiStepLR(self.optimizer, milestones=[50,75], gamma=0.1)

# 前30步时，学习率乘以0.1，之后恢复
scheduler = torch.optim.lr_scheduler.ConstantLR(self.optimizer, factor=0.1, total_iters=30)

# 初始学习率乘以0.1，之后20步线性恢复初始学习率
scheduler = torch.optim.lr_scheduler.LinearLR(self.optimizer, start_factor=0.1, total_iters=20)

# 初始学习率0.1， 每步学习率乘以0.9
scheduler = torch.optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.9)

# 循环学习率，循环半周期20步，最小学习率0.01， 最大学习率0.1
scheduler = torch.optim.lr_scheduler.CyclicLR(self.optimizer, base_lr=0.01, max_lr=0.1, step_size_up=20)

# 余弦调整学习率，半周期为10步，最小学习率为0.05
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=10, eta_min=0.05)

# 初始半周期为5，半周期系数为2，最小学习率为0.05
scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(self.optimizer, T_0=10, T_mult=2, eta_min=0.05)

# 学习率调整组合(顺序)
scheduler1 = torch.optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.9)
scheduler2 = torch.optim.lr_scheduler.MultiStepLR(self.optimizer, milestones=[60, 85], gamma=0.1)
# 前20步，使用scheduler1， 之后用scheduler2
scheduler = torch.optim.lr_scheduler.SequentialLR(self.optimizer, schedulers=[scheduler1, scheduler2], milestones=[20])

# 多个学习率调整叠加。
scheduler1 = torch.optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.9)
scheduler2 = torch.optim.lr_scheduler.ConstantLR(self.optimizer, factor=0.1, total_iters=20)
scheduler = torch.optim.lr_scheduler.ChainedScheduler([scheduler1, scheduler2])