#%% INTRODUCTION #https://github.com/gabrielloye/RNN-walkthrough/blob/master/main.ipynb # We'll be implementing a simple RNN character model with PyTorch to familiarize ourselves with the PyTorch library and get started with RNNs. You can run the code we’re using on FloydHub by clicking the button below and creating the project as well. # We'll start off by importing the main PyTorch package along with the Variable class used to store our data tensors and the nn package which we will use when building the model. In addition, we'll only be using numpy to pre-process our data as Torch works really well with numpy. import torch from torch import nn import numpy as np #%% DATA # First, we'll define the sentences that we want our model to output when fed with the first word or the first few characters. # Then we'll create a dictionary out of all the characters that we have in the sentences and map them to an integer. This will allow us to convert our input characters to their respective integers (char2int) and vice versa (int2char). text = ['hey how are you','good i am fine','have a nice day'] #text = ['hey how are you','good i am fine','have a nice day','Hello', 'Good morning', 'Good afternoon', 'Good evening', 'It’s nice to meet you', 'It’s a pleasure to meet you' ] # Join all the sentences together and extract the unique characters from the combined sentences chars = set(''.join(text)) # Creating a dictionary that maps integers to the characters int2char = dict(enumerate(chars)) # Creating another dictionary that maps characters to integers char2int = {char: ind for ind, char in int2char.items()} # in case you want to see what it is: print(char2int) #%% PADDING # Next, we'll be padding our input sentences to ensure that all the sentences are of the sample length. While RNNs are typically able to take in variably sized inputs, we will usually want to feed training data in batches to speed up the training process. In order to used batches to train on our data, we'll need to ensure that each sequence within the input data are of equal size. # Therefore, in most cases, padding can be done by filling up sequences that are too short with 0 values and trimming sequences that are too long. In our case, we'll be finding the length of the longest sequence and padding the rest of the sentences with blank spaces to match that length. maxlen = len(max(text, key=len)) print("The longest string has {} characters".format(maxlen)) # A simple loop that loops through the list of sentences and adds a ' ' whitespace until the length of the sentence matches the length of the longest sentence for i in range(len(text)): while len(text[i]) (Batch Size, Sequence Length, One-Hot Encoding Size)".format(input_seq.shape)) #%% PyTorch # Since we're done with all the data pre-processing, we can now move the data from numpy arrays to PyTorch's very own data structure - Torch Tensors input_seq = torch.from_numpy(input_seq) target_seq = torch.Tensor(target_seq) # Now we've reached the fun part of this project! We'll be defining the model using the Torch library, and this is where you can add or remove layers, be it fully connected layers, convolutational layers, vanilla RNN layers, LSTM layers, and many more! In this post, we'll be using the basic nn.rnn to demonstrate a simple example of how RNNs can be used. # Before we start building the model, let's use a build in feature in PyTorch to check the device we're running on (CPU or GPU). This implementation will not require GPU as the training is really simple. However, as you progress on to large datasets and models with millions of trainable parameters, using the GPU will be very important to speed up your training. # torch.cuda.is_available() checks and returns a Boolean True if a GPU is available, else it'll return False is_cuda = torch.cuda.is_available() # If we have a GPU available, we'll set our device to GPU. We'll use this device variable later in our code. if is_cuda: device = torch.device("cuda") print("GPU is available") else: device = torch.device("cpu") print("GPU not available, CPU used") #%% MODEL # To start building our own neural network model, we can define a class that inherits PyTorch’s base class (nn.module) for all neural network modules. After doing so, we can start defining some variables and also the layers for our model under the constructor. For this model, we’ll only be using 1 layer of RNN followed by a fully connected layer. The fully connected layer will be in-charge of converting the RNN output to our desired output shape. # We’ll also have to define the forward pass function under forward() as a class method. The order the forward function is sequentially executed, therefore we’ll have to pass the inputs and the zero-initialized hidden state through the RNN layer first, before passing the RNN outputs to the fully-connected layer. Note that we are using the layers that we defined in the constructor. # The last method that we have to define is the method that we called earlier to initialize the hidden state - init_hidden(). This basically creates a tensor of zeros in the shape of our hidden states. class Model(nn.Module): def __init__(self, input_size, output_size, hidden_dim, n_layers): super(Model, self).__init__() # Defining some parameters self.hidden_dim = hidden_dim self.n_layers = n_layers #Defining the layers # RNN Layer self.rnn = nn.RNN(input_size, hidden_dim, n_layers, batch_first=True) # Fully connected layer self.fc = nn.Linear(hidden_dim, output_size) def forward(self, x): batch_size = x.size(0) #Initializing hidden state for first input using method defined below hidden = self.init_hidden(batch_size) # Passing in the input and hidden state into the model and obtaining outputs out, hidden = self.rnn(x, hidden) # Reshaping the outputs such that it can be fit into the fully connected layer out = out.contiguous().view(-1, self.hidden_dim) out = self.fc(out) return out, hidden def init_hidden(self, batch_size): # This method generates the first hidden state of zeros which we'll use in the forward pass hidden = torch.zeros(self.n_layers, batch_size, self.hidden_dim).to(device) # We'll send the tensor holding the hidden state to the device we specified earlier as well return hidden #%% Hyper-Parameters # After defining the model above, we'll have to instantiate the model with the relevant parameters and define our hyperparamters as well. The hyperparameters we're defining below are: # 1.n_epochs: Number of Epochs --> This refers to the number of times our model will go through the entire training dataset # 2.lr: Learning Rate --> This affects the rate at which our model updates the weights in the cells each time backpropogation is done. A smaller learning rate means that the model changes the values of the weight with a smaller magnitude. A larger learning rate means that the weights are updated to a larger extent for each time step. Similar to other neural networks, we have to define the optimizer and loss function as well. We’ll be using CrossEntropyLoss as the final output is basically a classification task. # Instantiate (create) the model with the given hyperparameters model = Model(input_size=dict_size, output_size=dict_size, hidden_dim=12, n_layers=1) # We'll also set the model to the device that we defined earlier (default is CPU) model = model.to(device) # Define hyperparameters n_epochs = 100 lr=0.01 # Define Loss, Optimizer criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=lr) #%% Training # Now we can begin our training! As we only have a few sentences, this training process is very fast. However, as we progress, larger datasets and deeper models mean that the input data is much larger and the number of parameters within the model that we have to compute is much more. input_seq = input_seq.to(device) for epoch in range(1, n_epochs + 1): optimizer.zero_grad() # Clears existing gradients from previous epoch #input_seq = input_seq.to(device) output, hidden = model(input_seq) output = output.to(device) target_seq = target_seq.to(device) loss = criterion(output, target_seq.view(-1).long()) loss.backward() # Does backpropagation and calculates gradients optimizer.step() # Updates the weights accordingly if epoch%10 == 0: print('Epoch: {}/{}.............'.format(epoch, n_epochs), end=' ') print("Loss: {:.4f}".format(loss.item())) #%% Testing # Let’s test our model now and see what kind of output we will get. Before that, let’s define some helper function to convert our model output back to text. def predict(model, character): # One-hot encoding our input to fit into the model character = np.array([[char2int[c] for c in character]]) character = one_hot_encode(character, dict_size, character.shape[1], 1) character = torch.from_numpy(character) character = character.to(device) out, hidden = model(character) prob = nn.functional.softmax(out[-1], dim=0).data # Taking the class with the highest probability score from the output char_ind = torch.max(prob, dim=0)[1].item() return int2char[char_ind], hidden def sample(model, out_len, start='hey'): model.eval() # eval mode start = start.lower() # First off, run through the starting characters chars = [ch for ch in start] size = out_len - len(chars) # Now pass in the previous characters and get a new one for ii in range(size): char, h = predict(model, chars) chars.append(char) return ''.join(chars) print(" ") anOutput = sample(model, 15, 'hey') print(anOutput) # Check to see if the output is "good i am fine" # As we can see, the model is able to come up with the sentence ‘good i am fine ‘ if we feed it with the words ‘good’, achieving what we intended for it to do!