lr_scaler

zeus.utils.lr_scaler

Classes that enclose learning rate scaling rules.

SquareRootScaler dataclass

Square root scaling.

Parameters:

Name	Type	Description	Default
bs	int	The initial batch size	required
lr	float	The initial learning rate	required

Source code in zeus/utils/lr_scaler.py

@dataclass
class SquareRootScaler:
    """Square root scaling.

    Args:
        bs: The initial batch size
        lr: The initial learning rate
    """

    bs: int
    lr: float

    def compute_lr(self, new_bs: int) -> float:
        """Compute the scaled learning rate given the new batch size."""
        return self.lr * math.sqrt(new_bs / self.bs)

compute_lr

compute_lr(new_bs)

Compute the scaled learning rate given the new batch size.

Source code in zeus/utils/lr_scaler.py

def compute_lr(self, new_bs: int) -> float:
    """Compute the scaled learning rate given the new batch size."""
    return self.lr * math.sqrt(new_bs / self.bs)

LinearScaler dataclass

Linear scaling.

Parameters:

Name	Type	Description	Default
bs	int	The initial batch size	required
lr	float	The initial learning rate	required

Source code in zeus/utils/lr_scaler.py

@dataclass
class LinearScaler:
    """Linear scaling.

    Args:
        bs: The initial batch size
        lr: The initial learning rate
    """

    bs: int
    lr: float

    def compute_lr(self, new_bs: int) -> float:
        """Compute the scaled learning rate given the new batch size."""
        return self.lr * new_bs / self.bs

compute_lr

compute_lr(new_bs)

Compute the scaled learning rate given the new batch size.

Source code in zeus/utils/lr_scaler.py

def compute_lr(self, new_bs: int) -> float:
    """Compute the scaled learning rate given the new batch size."""
    return self.lr * new_bs / self.bs