init project

.gitignore

+.venv/

README.md

+# Gradient descent
+
+## Prerequisites
+
+### Create a `python` virtual environment
+
+```bash
+python3 -m venv .venv
+```
+
+### Activate the virtual environment
+
+```bash
+source .venv/bin/activate
+```
+
+### Installing the dependencies
+
+```bash
+pip3 install -r requirements.txt
+```
+
+## Running the programs
+
+```bash
+python3 gd.py
+```

gd.py

+import numpy as np
+from sympy import symbols, sin, cos, exp, sqrt, pi, lambdify, Function
+import pandas as pd
+from typing import Callable
+import matplotlib.pyplot as plt
+from matplotlib.colors import LogNorm
+
+
+def symb_grad_2d(f: Function) -> (Function, Function):
+    return f.diff(x), f.diff(y)
+
+
+def callable_grad_2d(f: Function) -> (Callable[[float, float], float],
+                                      Callable[[float, float], float]):
+    partialx, partialy = symb_grad_2d(f)
+    return (lambdify([x, y], partialx), lambdify([x, y], partialy))
+
+
+def callable_func(f: Function) -> Callable[[float, float], float]:
+    return lambdify([x, y], f, modules='numpy')
+
+
+def base_gd(f: Function, init_pt: list[float], lr: float) -> \
+        float:
+    df = pd.DataFrame(columns=['x', 'y', 'Cost', 'NormGrad'])
+    partialx, partialy = callable_grad_2d(f)
+
+    x, y = init_pt
+
+    iter = 0
+    grad = np.array([partialx(x, y), partialy(x, y)])
+
+    f_call = callable_func(f)
+
+    # while np.linalg.norm(grad) > 1e-6:
+    while iter < 1e4:
+        grad = np.array([partialx(x, y), partialy(x, y)])
+        df.loc[iter] = [x, y, f_call(x, y), np.linalg.norm(grad)]
+
+        step = -lr * grad
+
+        x += step[0]
+        y += step[1]
+
+        iter += 1
+
+    return df
+
+
+def momentum_gd(f: Function, init_pt: list[float], lr: float, momentum: float)\
+        -> float:
+    df = pd.DataFrame(columns=['x', 'y', 'Cost', 'NormGrad'])
+    partialx, partialy = callable_grad_2d(f)
+
+    x, y = init_pt
+
+    iter = 0
+    grad = np.array([partialx(x, y), partialy(x, y)])
+
+    f_call = callable_func(f)
+
+    prev_step = np.array([0, 0])
+    curr_step = np.array([0, 0])
+
+    # while np.linalg.norm(grad) > 1e-6:
+    while iter < 1e4:
+        grad = np.array([partialx(x, y), partialy(x, y)])
+        df.loc[iter] = [x, y, f_call(x, y), np.linalg.norm(grad)]
+
+        curr_step = -momentum * prev_step - lr * grad
+
+        x += curr_step[0]
+        y += curr_step[1]
+
+        prev_step = curr_step
+        iter += 1
+
+    return df
+
+
+def nesterov_gd(f: Function, init_pt: list[float], lr: float, momentum: float)\
+        -> float:
+    df = pd.DataFrame(columns=['x', 'y', 'Cost', 'NormGrad'])
+    partialx, partialy = callable_grad_2d(f)
+
+    x, y = init_pt
+
+    iter = 0
+    grad = np.array([partialx(x, y), partialy(x, y)])
+
+    f_call = callable_func(f)
+
+    prev_step = np.array([0, 0])
+    curr_step = np.array([0, 0])
+
+    # while np.linalg.norm(grad) > 1e-6:
+    while iter < 1e4:
+        grad = np.array([partialx(x, y), partialy(x, y)])
+        df.loc[iter] = [x, y, f_call(x, y), np.linalg.norm(grad)]
+
+        momentum_prev = -momentum * prev_step
+        offset_grad = np.array([partialx(x - momentum_prev[0], y - momentum_prev[1]),
+                               partialy(x - momentum_prev[0], y - momentum_prev[1])])
+
+        curr_step = momentum_prev - lr * offset_grad
+
+        x += curr_step[0]
+        y += curr_step[1]
+
+        prev_step = curr_step
+        iter += 1
+
+    return df
+
+
+if __name__ == "__main__":
+    x, y = symbols('x y')
+
+    # f: Function = x**2 + 5 * y**2
+    # f: Function = 1 - exp(-10 * x**2 - y**2)
+    # f: Function = x**2 * y - 2 * x * y**3 + 3 * x * y + 4
+
+    # Rosenbrock(x, y)
+    # f: Function = (1 - x)**2 + 100 * (y - x**2)**2
+
+    # Beale(x, y)
+    # f: Function = (1.5 - x + x * y)**2 + (2.25 - x + x *
+    #                                       y**2)**2 + (2.625 - x + x * y**3)**2
+
+    # Booth(x, y)
+    # f: Function = (x + 2 * y - 7)**2 + (2 * x + y - 5)**2
+
+    # Ackley(x, y)
+    f: Function = -20.0 * exp(-0.2 * sqrt(0.5 * (x**2 + y**2))) - \
+        exp(0.5 * (cos(2 * pi * x) + cos(2 * pi * y))) + exp(1) + 20
+
+    f_call = callable_func(f)
+
+    LR = 1e-1
+    MOMENTUM = 1e-1
+
+    plot_range = (10, 10)
+
+    # init_pt = [9, -8]
+    # init_pt = [1, 1]
+
+    init_pt = np.array([np.random.randint(-plot_range[0], plot_range[0] + 1),
+                        np.random.randint(-plot_range[1], plot_range[1] + 1)])
+
+    base = base_gd(f, init_pt, LR)
+    momentum = momentum_gd(f, init_pt, LR, MOMENTUM)
+    nesterov = nesterov_gd(f, init_pt, LR, MOMENTUM)
+
+    print(f"Base = {base.tail(1)}")
+    print(f"Momentum = {momentum.tail(1)}")
+    print(f"Nesterov = {nesterov.tail(1)}")
+
+    x = np.linspace(-plot_range[0], plot_range[0], 100)
+    y = np.linspace(-plot_range[1], plot_range[1], 100)
+
+    X, Y = np.meshgrid(x, y)
+    Z = f_call(X, Y)
+
+    fig = plt.figure(1)
+    ax = plt.axes(projection='3d')
+    ax.plot_surface(X, Y, Z, cmap='jet', rstride=1,
+                    cstride=1, norm=LogNorm(), alpha=0.2)
+    ax.set_xlabel('x')
+    ax.set_ylabel('y')
+    ax.set_zlabel('z')
+
+    ax.contour(X, Y, Z, 20, cmap='RdGy', offset=0)
+
+    ax.plot(base['x'], base['y'], '-o', color='red', label='base')
+    ax.plot(momentum['x'], momentum['y'], '-o',
+            color='violet', label='momentum')
+    ax.plot(nesterov['x'], nesterov['y'], '-o',
+            color='green', label='nesterov')
+    ax.legend()
+
+    plt.figure(2)
+    plt.contour(X, Y, Z, 20, cmap='RdGy')
+    plt.plot(base['x'], base['y'], '-o', color='red', label='base')
+    plt.plot(momentum['x'], momentum['y'], '-o',
+             color='violet', label='momentum')
+    plt.plot(nesterov['x'], nesterov['y'], '-o',
+             color='green', label='nesterov')
+    plt.legend()
+    plt.xlabel('x')
+    plt.ylabel('y')
+
+    plt.show()

requirements.txt

+contourpy==1.2.1
+cycler==0.12.1
+fonttools==4.51.0
+kiwisolver==1.4.5
+matplotlib==3.8.4
+mpmath==1.3.0
+numpy==1.26.4
+packaging==24.0
+pandas==2.2.2
+pillow==10.3.0
+pyparsing==3.1.2
+PyQt6==6.7.0
+PyQt6-Qt6==6.7.0
+PyQt6-sip==13.6.0
+python-dateutil==2.9.0.post0
+pytz==2024.1
+six==1.16.0
+sympy==1.12
+tzdata==2024.1