תרגול 11 - CNN ואתחול

Setup¶

## Importing packages
import os # A build in package for interacting with the OS. For example to create a folder.
import numpy as np  # Numerical package (mainly multi-dimensional arrays and linear algebra)
import pandas as pd  # A package for working with data frames
import matplotlib.pyplot as plt  # A plotting package
import imageio  # A package to read and write image (is used here to save gif images)
import tabulate  # A package from pretty printing tables
from graphviz import Digraph  # A package for plothing graphs (of nodes and edges)

## Setup matplotlib to output figures into the notebook
## - To make the figures interactive (zoomable, tooltip, etc.) use ""%matplotlib notebook" instead
%matplotlib inline

## Setting some nice matplotlib defaults
plt.rcParams['figure.figsize'] = (4.5, 4.5)  # Set default plot's sizes
plt.rcParams['figure.dpi'] = 120  # Set default plot's dpi (increase fonts' size)
plt.rcParams['axes.grid'] = True  # Show grid by default in figures

## Auxiliary function for prining equations, pandas tables and images in cells output
from IPython.core.display import display, HTML, Latex, Markdown

## Create output folder
if not os.path.isdir('./output'):
    os.mkdir('./output')

Initilization¶

Exploding Gradients¶

rang_gen = np.random.RandomState(2)

width = 5

weights = []
for _ in range(4):
    weights.append(10 * rang_gen.randn(width, width))
weights.append(rang_gen.randn(width, 1))

z = np.array(rang_gen.randn(width))

print(f'x=[' + ','.join([f'{val:.2f}' for val in z]) + r']^{\top}')
np.set_printoptions(precision=2)
for i, w in enumerate(weights):
    z = np.maximum(z @ w, 0)
#     print(z.mean())
    print(f'z_{i+1}=[' + ','.join([f'{val:.2f}' for val in z]) + r']^{\top}')

x=[2.59,1.84,0.44,-0.72,-0.58]^{\top}
z_1=[0.00,18.93,0.00,29.21,0.00]^{\top}
z_2=[0.00,647.54,0.00,527.02,498.82]^{\top}
z_3=[5263.19,0.00,0.00,4494.22,0.00]^{\top}
z_4=[100556.40,0.00,4364.23,99805.43,0.00]^{\top}
z_5=[155078.15]^{\top}

Vanishing Gradients¶

rang_gen = np.random.RandomState(2)
weights = []
for _ in range(4):
    weights.append(0.1 * rang_gen.randn(width, width))
weights.append(rang_gen.randn(width, 1))

z = np.array(rang_gen.randn(width))

print(f'x=[' + ','.join([f'{val:.8f}' for val in z]) + r']^{\top}')
np.set_printoptions(precision=2)
for i, w in enumerate(weights):
    z = np.maximum(z @ w, 0)
#     print(z.mean())
    print(f'z_{i+1}=[' + ','.join([f'{val:.5f}' for val in z]) + r']^{\top}')

x=[2.58529487,1.83533272,0.44068987,-0.71925384,-0.58341459]^{\top}
z_1=[0.00000,0.18927,0.00000,0.29210,0.00000]^{\top}
z_2=[0.00000,0.06475,0.00000,0.05270,0.04988]^{\top}
z_3=[0.00526,0.00000,0.00000,0.00449,0.00000]^{\top}
z_4=[0.00101,0.00000,0.00004,0.00100,0.00000]^{\top}
z_5=[0.00155]^{\top}