From 59119180234984ea7c3322e3e6a9ee1515bff112 Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Fri, 28 Aug 2026 14:02:25 +0530 Subject: [PATCH] Add non-linearity to hidden state in char-RNN generation tutorial --- intermediate_source/char_rnn_generation_tutorial.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/intermediate_source/char_rnn_generation_tutorial.py b/intermediate_source/char_rnn_generation_tutorial.py index 50a6afa11b7..73bd628ecfc 100644 --- a/intermediate_source/char_rnn_generation_tutorial.py +++ b/intermediate_source/char_rnn_generation_tutorial.py @@ -139,7 +139,10 @@ def readLines(filename): # letter. # # I added a second linear layer ``o2o`` (after combining hidden and -# output) to give it more muscle to work with. There's also a dropout +# output) to give it more muscle to work with. The hidden state is also +# passed through a ``tanh`` non-linearity before being carried to the +# next time step, since otherwise it would just be a linear combination +# of the previous hidden state and the current input. There's also a dropout # layer, which `randomly zeros parts of its # input `__ with a given probability # (here 0.1) and is usually used to fuzz inputs to prevent overfitting. @@ -162,12 +165,13 @@ def __init__(self, input_size, hidden_size, output_size): self.i2h = nn.Linear(n_categories + input_size + hidden_size, hidden_size) self.i2o = nn.Linear(n_categories + input_size + hidden_size, output_size) self.o2o = nn.Linear(hidden_size + output_size, output_size) + self.tanh = nn.Tanh() self.dropout = nn.Dropout(0.1) self.softmax = nn.LogSoftmax(dim=1) def forward(self, category, input, hidden): input_combined = torch.cat((category, input, hidden), 1) - hidden = self.i2h(input_combined) + hidden = self.tanh(self.i2h(input_combined)) output = self.i2o(input_combined) output_combined = torch.cat((hidden, output), 1) output = self.o2o(output_combined)