12  Recurrent Neural Networks (RNN)

Recurrent neural networks are used to model sequential data, i.e. a temporal sequence that exhibits temporal dynamic behavior. Here is a good introduction to the topic:

12.1 Case Study: Predicting drought

We will use a subset of the data explained in this github repository

utils::download.file("https://www.dropbox.com/s/radyscnl5zcf57b/weather_soil.RDS?raw=1", destfile = "weather_soil.RDS")
data = readRDS("weather_soil.RDS")
X = data$train # Features of the last 180 days
dim(X)

[1] 999 180  21

# 999 batches of 180 days with 21 features each
Y = data$target
dim(Y)

[1] 999   6

# 999 batches of 6 week drought predictions

# let's visualize drought over 24 months:
# -> We have to take 16 batches (16*6 = 96 weaks ( = 24 months) )
plot(as.vector(Y[1:16,]), type = "l", xlab = "week", ylab = "Drought")

library(torch)

holdout = 700:999
X_train = X[-holdout,,]
X_test = X[holdout,,]

Y_train = Y[-holdout,]
Y_test = Y[holdout,]



net = nn_module(
  initialize = function() {
    self$lstm = nn_lstm(input_size = 21L, hidden_size = 50L ,batch_first = TRUE)
    self$linear = nn_linear(50,  6L)
  }, 
  forward = function(x) {
    x = self$lstm(x)
    # two outputs -> [batch, seq_len, hidden] , last hidden state
    x = x[[1]][,dim(x[[1]])[2], ]
    return(self$linear(x))
  }
)

# training loop
model = net()

train_dl = dataloader(tensor_dataset(torch_tensor(X_train), torch_tensor(Y_train)), batch_size = 100L, shuffle = TRUE )

opt = optim_adam(params = model$parameters, lr = 0.01)
epochs = 100L
overall_train_loss = c()
for(e in 1:epochs) {
  losses = losses_val = c()
  model$train() # -> dropout is on
  coro::loop(
    for(batch in train_dl) {
      x = batch[[1]] # Feature matrix/tensor
      y = batch[[2]] # Response matrix/tensor
      opt$zero_grad() # reset optimizer
      pred = model(x)
      loss = nnf_mse_loss(pred, y)
      loss$backward()
      opt$step() # update weights
      losses = c(losses, loss$item())
    }
  )
  overall_train_loss = c(overall_train_loss, mean(losses))
  cat(sprintf("Loss at epoch: %d train: %3f\n", e, mean(losses)))

}

Loss at epoch: 1 train: 1.994906
Loss at epoch: 2 train: 0.856170
Loss at epoch: 3 train: 0.476273
Loss at epoch: 4 train: 0.380085
Loss at epoch: 5 train: 0.376748
Loss at epoch: 6 train: 0.357207
Loss at epoch: 7 train: 0.341953
Loss at epoch: 8 train: 0.346024
Loss at epoch: 9 train: 0.336112
Loss at epoch: 10 train: 0.333987
Loss at epoch: 11 train: 0.331322
Loss at epoch: 12 train: 0.296810
Loss at epoch: 13 train: 0.296099
Loss at epoch: 14 train: 0.283915
Loss at epoch: 15 train: 0.278939
Loss at epoch: 16 train: 0.271180
Loss at epoch: 17 train: 0.276021
Loss at epoch: 18 train: 0.267254
Loss at epoch: 19 train: 0.255515
Loss at epoch: 20 train: 0.243821
Loss at epoch: 21 train: 0.241473
Loss at epoch: 22 train: 0.245328
Loss at epoch: 23 train: 0.252983
Loss at epoch: 24 train: 0.275309
Loss at epoch: 25 train: 0.261029
Loss at epoch: 26 train: 0.234615
Loss at epoch: 27 train: 0.220224
Loss at epoch: 28 train: 0.203021
Loss at epoch: 29 train: 0.181731
Loss at epoch: 30 train: 0.175554
Loss at epoch: 31 train: 0.175870
Loss at epoch: 32 train: 0.195022
Loss at epoch: 33 train: 0.183308
Loss at epoch: 34 train: 0.168748
Loss at epoch: 35 train: 0.168387
Loss at epoch: 36 train: 0.153121
Loss at epoch: 37 train: 0.149006
Loss at epoch: 38 train: 0.151347
Loss at epoch: 39 train: 0.150211
Loss at epoch: 40 train: 0.135393
Loss at epoch: 41 train: 0.126832
Loss at epoch: 42 train: 0.116456
Loss at epoch: 43 train: 0.113194
Loss at epoch: 44 train: 0.106261
Loss at epoch: 45 train: 0.101918
Loss at epoch: 46 train: 0.097186
Loss at epoch: 47 train: 0.093761
Loss at epoch: 48 train: 0.091179
Loss at epoch: 49 train: 0.087262
Loss at epoch: 50 train: 0.086706
Loss at epoch: 51 train: 0.093703
Loss at epoch: 52 train: 0.092632
Loss at epoch: 53 train: 0.089517
Loss at epoch: 54 train: 0.087657
Loss at epoch: 55 train: 0.082732
Loss at epoch: 56 train: 0.078252
Loss at epoch: 57 train: 0.074534
Loss at epoch: 58 train: 0.076109
Loss at epoch: 59 train: 0.072607
Loss at epoch: 60 train: 0.070278
Loss at epoch: 61 train: 0.069510
Loss at epoch: 62 train: 0.065573
Loss at epoch: 63 train: 0.066073
Loss at epoch: 64 train: 0.062593
Loss at epoch: 65 train: 0.060189
Loss at epoch: 66 train: 0.058890
Loss at epoch: 67 train: 0.057784
Loss at epoch: 68 train: 0.057428
Loss at epoch: 69 train: 0.054572
Loss at epoch: 70 train: 0.056867
Loss at epoch: 71 train: 0.055438
Loss at epoch: 72 train: 0.052952
Loss at epoch: 73 train: 0.052330
Loss at epoch: 74 train: 0.052945
Loss at epoch: 75 train: 0.055448
Loss at epoch: 76 train: 0.052313
Loss at epoch: 77 train: 0.048622
Loss at epoch: 78 train: 0.050235
Loss at epoch: 79 train: 0.047862
Loss at epoch: 80 train: 0.049148
Loss at epoch: 81 train: 0.047287
Loss at epoch: 82 train: 0.053710
Loss at epoch: 83 train: 0.054321
Loss at epoch: 84 train: 0.054545
Loss at epoch: 85 train: 0.054815
Loss at epoch: 86 train: 0.050959
Loss at epoch: 87 train: 0.093483
Loss at epoch: 88 train: 0.098891
Loss at epoch: 89 train: 0.093983
Loss at epoch: 90 train: 0.092342
Loss at epoch: 91 train: 0.096662
Loss at epoch: 92 train: 0.092818
Loss at epoch: 93 train: 0.096260
Loss at epoch: 94 train: 0.080524
Loss at epoch: 95 train: 0.072129
Loss at epoch: 96 train: 0.061361
Loss at epoch: 97 train: 0.053054
Loss at epoch: 98 train: 0.057596
Loss at epoch: 99 train: 0.056570
Loss at epoch: 100 train: 0.054325

preds = as.matrix(model(torch_tensor(X_test)))





matplot(cbind(as.vector(preds[1:48,]),  
              as.vector(Y_test[1:48,])), 
        col = c("darkblue", "darkred"),
        type = "l", 
        pch = c(15, 16),
        xlab = "week", ylab = "Drought")
legend("topright", bty = "n", 
       col = c("darkblue", "darkred"),
      pch = c(15, 16), 
      legend = c("Prediction", "True Values"))