import numpy as np
import matplotlib.pyplot as plt


def sigmoid(x):
    return 1 / (1 + np.exp(-x))
def BatchNormalization(x):
    return x - np.mean(x) / np.std(x)


x = np.random.randn(1000,100)
node_num = 100
hidden_layer_size = 5
activations = {}

for i in range(hidden_layer_size):
    if i != 0:
        x = activations[i-1]

    w = np.random.randn(node_num, node_num) / np.sqrt(node_num) #Xavier
    a = np.dot(x,w)
    b = BatchNormalization(a)
    z = sigmoid(b)
    activations[i] = z

for i, a in activations.items():
    plt.subplot(1, len(activations), i+1)
    plt.title(str(i+1) + '-layer')
    plt.hist(a.flatten(),30,range=(0,1))
plt.show()
ccb0dc4f2ed4cfa71ad3d7440854076a1acd580c
<mask token>


def sigmoid(x):
    return 1 / (1 + np.exp(-x))


def BatchNormalization(x):
    return x - np.mean(x) / np.std(x)


<mask token>

<mask token>


def sigmoid(x):
    return 1 / (1 + np.exp(-x))


def BatchNormalization(x):
    return x - np.mean(x) / np.std(x)


<mask token>
for i in range(hidden_layer_size):
    if i != 0:
        x = activations[i - 1]
    w = np.random.randn(node_num, node_num) / np.sqrt(node_num)
    a = np.dot(x, w)
    b = BatchNormalization(a)
    z = sigmoid(b)
    activations[i] = z
for i, a in activations.items():
    plt.subplot(1, len(activations), i + 1)
    plt.title(str(i + 1) + '-layer')
    plt.hist(a.flatten(), 30, range=(0, 1))
plt.show()

<mask token>


def sigmoid(x):
    return 1 / (1 + np.exp(-x))


def BatchNormalization(x):
    return x - np.mean(x) / np.std(x)


x = np.random.randn(1000, 100)
node_num = 100
hidden_layer_size = 5
activations = {}
for i in range(hidden_layer_size):
    if i != 0:
        x = activations[i - 1]
    w = np.random.randn(node_num, node_num) / np.sqrt(node_num)
    a = np.dot(x, w)
    b = BatchNormalization(a)
    z = sigmoid(b)
    activations[i] = z
for i, a in activations.items():
    plt.subplot(1, len(activations), i + 1)
    plt.title(str(i + 1) + '-layer')
    plt.hist(a.flatten(), 30, range=(0, 1))
plt.show()

import numpy as np
import matplotlib.pyplot as plt


def sigmoid(x):
    return 1 / (1 + np.exp(-x))


def BatchNormalization(x):
    return x - np.mean(x) / np.std(x)


x = np.random.randn(1000, 100)
node_num = 100
hidden_layer_size = 5
activations = {}
for i in range(hidden_layer_size):
    if i != 0:
        x = activations[i - 1]
    w = np.random.randn(node_num, node_num) / np.sqrt(node_num)
    a = np.dot(x, w)
    b = BatchNormalization(a)
    z = sigmoid(b)
    activations[i] = z
for i, a in activations.items():
    plt.subplot(1, len(activations), i + 1)
    plt.title(str(i + 1) + '-layer')
    plt.hist(a.flatten(), 30, range=(0, 1))
plt.show()

import numpy as np
import matplotlib.pyplot as plt


def sigmoid(x):
    return 1 / (1 + np.exp(-x))
def BatchNormalization(x):
    return x - np.mean(x) / np.std(x)


x = np.random.randn(1000,100)
node_num = 100
hidden_layer_size = 5
activations = {}

for i in range(hidden_layer_size):
    if i != 0:
        x = activations[i-1]

    w = np.random.randn(node_num, node_num) / np.sqrt(node_num) #Xavier
    a = np.dot(x,w)
    b = BatchNormalization(a)
    z = sigmoid(b)
    activations[i] = z

for i, a in activations.items():
    plt.subplot(1, len(activations), i+1)
    plt.title(str(i+1) + '-layer')
    plt.hist(a.flatten(),30,range=(0,1))
plt.show()