diff --git a/uebungen/uebung3.py b/uebungen/uebung3.py index d15f3be..a0b7b8a 100644 --- a/uebungen/uebung3.py +++ b/uebungen/uebung3.py @@ -1,42 +1,110 @@ +from typing import Tuple + import numpy as np +import pandas as pd +import matplotlib.pyplot as plt -visible = np.ones((10,1)) -hidden = np.ones((5,1)) -visible_bias = np.ones((len(visible), 1)) * 0.1 -hidden_bias = np.ones((len(hidden),1)) * 0.1 +def load_data(): + df_orig_train = pd.read_csv('mnist_test_final.csv') + df_digits = df_orig_train.drop('label',axis=1) -weights = np.random.rand(len(visible), len(hidden)) -phases = 1 + return df_digits.to_numpy() -learnrate = 0.2 +mnist = load_data() def sigmoid(x): - return 1 / (1 + np.exp(-x)) # Sigmoidfunktion + return 1.0 / (1.0 + np.exp(-x)) # Sigmoidfunktion +class RBM: -for i in range(1): - activation = sigmoid(np.matmul(visible.T, weights) + hidden_bias) + def __init__(self, visible_size: int, hidden_size: int, learnrate: float=0.1) -> None: + self.learnrate = learnrate + self.visible_size = visible_size + self.hidden_size = hidden_size + self.k = 2 + self.epochs = 10 - # 2. Computer outer product vh - positive_gradient = np.matmul(visible, hidden.T) + self.reset() - t = sigmoid(np.matmul(weights, hidden) + visible_bias) - reconstructed = sigmoid(np.matmul(weights, hidden) + visible_bias) + def reset(self) -> None: + self.weights = np.random.randn(self.visible_size, self.hidden_size) + self.visible_bias = np.ones(self.visible_size) * 0.1 + self.hidden_bias = np.ones(self.hidden_size) * 0.1 - # 4. Computer outer product v'h' - negative_gradient = np.matmul(reconstructed.T, activation) - # 5. Update weight matrix using gradients - - delta_weights = learnrate * (positive_gradient - negative_gradient) - - # 6. Update bias for visible and hidden layer + def activate(self, v0): + return sigmoid(np.matmul(v0.T, self.weights) + self.hidden_bias) + + def reactivate(self, h0): + return sigmoid(np.matmul(self.weights, h0.T) + self.visible_bias) - delta_visible_bias = learnrate * (visible - reconstructed) - delta_hidden_bias = learnrate * (hidden - activation) + def contrastive_divergence(self, v0, h0, v1, h1): + postive_gradient = np.outer(v0, h0) + negative_gradient = np.outer(v1, h0) + + self.weights += self.learnrate * (postive_gradient - negative_gradient) + + return self.weights + + def gibbs(self, v0): + for _ in range(self.k): + hidden_probs = self.activate(v0) + h0 = np.random.rand(len(v0), self.hidden_size) < hidden_probs + visible_probs = self.sigmoid(np.dot(h0, self.weights.T) + self.visible_bias) + v0 = np.random.rand(len(v0), self.visible_size) < visible_probs + return v0, hidden_probs + + def train(self, v0): + for _ in range(self.epochs): + h0 = self.activate(v0) # Aktivieren versteckter Schicht + v1 = self.reactivate(h0) # Reaktivieren sichtbarer Schicht + h1 = self.activate(v1) + + self.contrastive_divergence(v0, h0, v1, h1) + + self.visible_bias += self.learnrate * (v0 - v1) + self.hidden_bias += self.learnrate * (h0 - h1) + return h0, v1 + + def run(self, v0 : np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """run Runs the Restricted Boltzmann machine on some input vector v0. + + Args: + v0 (np.ndarray): 1-dimensional Input vector + + Returns: + Tuple[np.ndarray, np.ndarray]: (hidden activation, visible reactivation) + """ + h0 = self.activate(v0) + v1 = self.reactivate(h0) + + return h0, v1 +def validate(idx): + + test = mnist[idx].flatten() + #rbm.reset() + rbm.train(test) + (hid, out) = rbm.run(test) + + return (hid.reshape((28, 28)), out.reshape((28,28))) +rbm = RBM(28**2, 28**2, 0.1) + +rows, columns = (4,4) +fig = plt.figure(figsize=(10, 7)) +for i in range((rows * columns)): + if i % 2 == 0: + (hid, out) = validate(i) + fig.add_subplot(rows, columns, i+1) + plt.imshow(hid, cmap='gray') + fig.add_subplot(rows, columns, i+2) + plt.imshow(out, cmap='gray') + plt.axis('off') + +plt.show() +