Activation Functions¶

Libraries¶

In [1]:

# Standard libraries
import math
from typing import Callable

# 3pps
import matplotlib.pyplot as plt
import numpy as np

# Standard libraries import math from typing import Callable # 3pps import matplotlib.pyplot as plt import numpy as np

Functions¶

In [2]:

def plot_function(name_function: str, steps: np.ndarray, function: Callable) -> None:
    plt.title(f"{name_function} function")
    plt.plot(steps, function(steps))
    plt.grid()
    plt.show()

def plot_function(name_function: str, steps: np.ndarray, function: Callable) -> None: plt.title(f"{name_function} function") plt.plot(steps, function(steps)) plt.grid() plt.show()

In [3]:

def sigmoid(input: np.ndarray) -> np.ndarray:
    return 1/(1+np.exp((-1) * input))


def tanh(input: np.ndarray) -> np.ndarray:
    return (np.exp(input) - np.exp(-input)) / (np.exp(input) + np.exp(-input))


def relu(input: np.ndarray) -> np.ndarray:
    return [max(0, elem) for elem in input]


def leaky_relu(input: np.ndarray, alpha: float = 0.1) -> np.ndarray:
    return [max(0.1 * elem, elem) for elem in input]


def elu(input: np.ndarray, alpha: float = 0.5) -> np.ndarray:
    return [alpha * (np.exp(elem) - 1) if elem < 0 else elem for elem in input]


def swish(input: np.ndarray) -> np.ndarray:
    return input * sigmoid(input)


def gelu(input: np.ndarray) -> np.ndarray:
    return 0.5 * input * (1 + tanh(math.sqrt(2/math.pi) * (input + 0.044715 * input ** 3)))

def sigmoid(input: np.ndarray) -> np.ndarray: return 1/(1+np.exp((-1) * input)) def tanh(input: np.ndarray) -> np.ndarray: return (np.exp(input) - np.exp(-input)) / (np.exp(input) + np.exp(-input)) def relu(input: np.ndarray) -> np.ndarray: return [max(0, elem) for elem in input] def leaky_relu(input: np.ndarray, alpha: float = 0.1) -> np.ndarray: return [max(0.1 * elem, elem) for elem in input] def elu(input: np.ndarray, alpha: float = 0.5) -> np.ndarray: return [alpha * (np.exp(elem) - 1) if elem < 0 else elem for elem in input] def swish(input: np.ndarray) -> np.ndarray: return input * sigmoid(input) def gelu(input: np.ndarray) -> np.ndarray: return 0.5 * input * (1 + tanh(math.sqrt(2/math.pi) * (input + 0.044715 * input ** 3)))

Main¶

In [4]:

steps = np.arange(-10, 10, 0.1)

plot_function(name_function="Sigmoid", steps=steps, function=sigmoid)
plot_function(name_function="Tanh", steps=steps, function=tanh)
plot_function(name_function="ReLU", steps=steps, function=relu)
plot_function(name_function="LeakyReLU", steps=steps, function=leaky_relu)
plot_function(name_function="ELU", steps=steps, function=elu)
plot_function(name_function="Swish", steps=steps, function=swish)
plot_function(name_function="GELU", steps=steps, function=gelu)

steps = np.arange(-10, 10, 0.1) plot_function(name_function="Sigmoid", steps=steps, function=sigmoid) plot_function(name_function="Tanh", steps=steps, function=tanh) plot_function(name_function="ReLU", steps=steps, function=relu) plot_function(name_function="LeakyReLU", steps=steps, function=leaky_relu) plot_function(name_function="ELU", steps=steps, function=elu) plot_function(name_function="Swish", steps=steps, function=swish) plot_function(name_function="GELU", steps=steps, function=gelu)

In [ ]: