Download No description has been provided for this image

תרגול 2 - רגרסיה לינארית

Setup

In [ ]:
## Importing packages
import os # A build in package for interacting with the OS. For example to create a folder.
import numpy as np  # Numerical package (mainly multi-dimensional arrays and linear algebra)
import pandas as pd  # A package for working with data frames
import matplotlib.pyplot as plt  # A plotting package
import imageio  # A package to read and write image (is used here to save gif images)

## Setup matplotlib to output figures into the notebook
## - To make the figures interactive (zoomable, tooltip, etc.) use ""%matplotlib notebook" instead
%matplotlib inline

## Setting some nice matplotlib defaults
plt.rcParams['figure.figsize'] = (4.5, 4.5)  # Set default plot's sizes
plt.rcParams['figure.dpi'] = 120  # Set default plot's dpi (increase fonts' size)
plt.rcParams['axes.grid'] = True  # Show grid by default in figures

## Auxiliary function for prining equations, pandas tables and images in cells output
from IPython.core.display import display, HTML, Latex, Markdown

## Create output folder
if not os.path.isdir('./output'):
    os.mkdir('./output')

Ex. 2.2

In [ ]:
x = np.array([-1, 2, 4])
y = np.array([2.5, 2, 1])

x_grid = np.arange(-2, 5, 0.1)
In [ ]:
## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_dataset.png')
No description has been provided for this image

Section 1

In [ ]:
## Defining augmentation
aug_func1 = lambda x: np.stack((np.ones(x.shape[0]), x), axis=1)

## Augment the dataset
x_aug = aug_func1(x)

## Calcualting theta
theta1 = np.linalg.inv(x_aug.T @ x_aug) @ (x_aug.T @ y)

## Printing derivation
denom = np.linalg.det(x_aug.T @ x_aug).round()
display(
    Markdown('##### Derivation'),
    Latex(r'$(X^{\top}X)^{-1}=\frac{1}{' + f'{denom}' r'}\times$'),
    np.linalg.inv(x_aug.T @ x_aug) * denom,
    Latex(r'$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{' + f'{denom}}}${theta1 * denom}={theta1}'),
    )

## Defineing the predictor
h1 = lambda x: aug_func1(x) @ theta1

## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, h1(x_grid), label=f'Liner model')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_1.png')
Derivation
$(X^{\top}X)^{-1}=\frac{1}{38.0}\times$ 
array([[21., -5.],
       [-5.,  3.]])
$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{38.0}$[ 88. -11.]=[ 2.31578947 -0.28947368]
No description has been provided for this image

Section 2

In [ ]:
## Defining augmentation
aug_func2 = lambda x: np.stack((np.ones(x.shape[0]), x, x ** 2), axis=1)

## Augment the dataset
x_aug = aug_func2(x)

## Calcualting theta
theta2 = np.linalg.inv(x_aug.T @ x_aug) @ (x_aug.T @ y)

## Printing derivation
denom = np.linalg.det(x_aug.T @ x_aug).round() / 30
display(
    Markdown('##### Derivation'),
    Latex(r'$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{' + f'{denom}}}${theta2 * denom}={theta2}'),
    )

## Defineing the predictor
h2 = lambda x: aug_func2(x) @ theta2

## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, h2(x_grid), label=f'2nd order')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_2.png')
Derivation
$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{30.0}$[74. -3. -2.]=[ 2.46666667 -0.1 -0.06666667]
No description has been provided for this image

Section 2_1

In [ ]:
## Defining augmentation
aug_func3_1 = lambda x: np.stack((x, x ** 2, x** 3), axis=1)

## Augment the dataset
x_aug = aug_func3_1(x)

## Calcualting theta
theta3_1 = np.linalg.inv(x_aug.T @ x_aug) @ (x_aug.T @ y)

## Printing derivation
denom = np.linalg.det(x_aug.T @ x_aug).round() / 480
display(
    Markdown('##### Derivation'),
    Latex(r'$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{' + f'{denom}}}${theta3_1 * denom}={theta3_1}'),
    )

## Defineing the predictor
h3_1 = lambda x: aug_func3_1(x) @ theta3_1

## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, h3_1(x_grid), label=f'3nd order - 1')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_3_2_3_1.png')
Derivation
$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{120.0}$[-86. 177. -37.]=[-0.71666667 1.475 -0.30833333]
No description has been provided for this image

Section 2_2

In [ ]:
## Defining augmentation
aug_func3_2 = lambda x: np.stack((np.ones(x.shape[0]), x, x** 3), axis=1)

## Augment the dataset
x_aug = aug_func3_2(x)

## Calcualting theta
theta3_2 = np.linalg.inv(x_aug.T @ x_aug) @ (x_aug.T @ y)

## Printing derivation
denom = np.linalg.det(x_aug.T @ x_aug).round() / 150
display(
    Markdown('##### Derivation'),
    Latex(r'$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{' + f'{denom}}}${theta3_2 * denom}={theta3_2}'),
    )

## Defineing the predictor
h3_2 = lambda x: aug_func3_2(x) @ theta3_2

## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, h3_2(x_grid), label=f'3nd order - 2')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_3_2.png')
Derivation
$\boldsymbol{\theta}^*_{\mathcal{D}}=\frac{1}{150.0}$[354. -19. -2.]=[ 2.36 -0.12666667 -0.01333333]
No description has been provided for this image

Section 4

In [ ]:
## Defining augmentation
feat1 = lambda x: np.exp(-(x+1) ** 2 / 1.5 ** 2 / 2)
feat2 = lambda x: np.exp(-(x-2) ** 2 / 2)
feat3 = lambda x: np.exp(-(x-4) ** 2 / 2)

aug_func4 = lambda x: np.stack((feat1(x), feat2(x), feat3(x)), axis=1)

## Augment the dataset
x_aug = aug_func4(x)

## Calcualting theta
theta4 = np.linalg.inv(x_aug.T @ x_aug) @ (x_aug.T @ y)

## Printing derivation
display(
    Markdown('##### Derivation'),
    Latex(r'$X$'),
    x_aug,
    Latex(r'$\boldsymbol{\theta}^*_{\mathcal{D}}=$' + f'{theta4}'),
    )

## Defineing the predictor
h4 = lambda x: aug_func4(x) @ theta4

## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, h4(x_grid), label=f'Gaussian')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_4.png')
Derivation
$X$ 
array([[1.00000000e+00, 1.11089965e-02, 3.72665317e-06],
       [1.35335283e-01, 1.00000000e+00, 1.35335283e-01],
       [3.86592014e-03, 1.35335283e-01, 1.00000000e+00]])
$\boldsymbol{\theta}^*_{\mathcal{D}}=$[2.48268357 1.55851414 0.77948019]
No description has been provided for this image
In [ ]:
## Ploting
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, feat1(x_grid) * theta4[0])
ax.plot(x_grid, feat2(x_grid) * theta4[1])
ax.plot(x_grid, feat3(x_grid) * theta4[2])
ax.plot(x_grid, h4(x_grid), '--', color='gray')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_4_decomp.png')
No description has been provided for this image

Section 5

In [ ]:
## Ploting all
fig, ax = plt.subplots(figsize=(4.5, 3))
ax.plot(x, y, 'x', ms=10, mew=3, label=f'Dataset')
ax.plot(x_grid, h1(x_grid), label=f'Liner model')
ax.plot(x_grid, h2(x_grid), label=f'2nd order')
ax.plot(x_grid, h3_1(x_grid), label=f'3nd order - 1')
ax.plot(x_grid, h3_2(x_grid), label=f'3nd order - 2')
ax.plot(x_grid, h4(x_grid), label=f'Gaussian')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend(loc='lower left')
ax.set_xlim(-2, 5)
ax.set_ylim(-1, 4)
plt.tight_layout()
fig.savefig('./output/ex_2_2_all.png')
No description has been provided for this image
In [ ]: