     float * this_layer_outputs = layer_outputs;
     float * next_layer_outputs;
 
+    int layer_weights_offset = 0;
+    int next_layer_weights_offset = 0;
+    int layer_bias_offset = 0;
+    int next_layer_bias_offset = 0;
+    
     for (unsigned l = 0;
          l < num_layers;
          ++l,
              __syncthreads(),
              last_layer_outputs = this_layer_outputs,
-             this_layer_outputs = next_layer_outputs) {
-
+             this_layer_outputs = next_layer_outputs,
+             layer_weights_offset = next_layer_weights_offset,
+             layer_bias_offset = next_layer_bias_offset) {
+        
         // Get information about the layer:
         int ni = architecture[l];
         int no = architecture[l + 1];
 
-        const float * layer_weights = w[l];
         int w_stride = w_strides[l];
 
         next_layer_outputs = this_layer_outputs + no;
+        next_layer_weights_offset += ni * w_stride;
+        next_layer_bias_offset += no;
 
 #if defined(__DEVICE_EMULATION__) && 0
         if (tid == 0)

        int ni = architecture[l];
        int no = architecture[l + 1];

        const float * layer_weights = w[l];
        int w_stride = w_strides[l];

        UpdateFloat * layer_updates = w_updates[l];
        UpdateFloat * layer_bias_updates  = b_updates[l];

    vector<int> architecture;
    DeviceData<int> d_architecture;

    DeviceData<float> d_weights;

    DeviceData<float> d_biases;

    vector<int> w_strides;
    DeviceData<int> d_w_strides;
