         } else {
           VLOG(1) << "Grad scale: " << FlattenToString(fp16_scale, 1, place);
         }
-        if (nranks > 1) {
-          PADDLE_ENFORCE_GPU_SUCCESS(
-              platform::dynload::ncclAllReduce(fp32_square_grad_norm,
-                                               fp32_square_grad_norm,
-                                               1,
-                                               ncclFloat32,
-                                               ncclSum,
-                                               global_comm,
-                                               stream));
-        }
         // (3) Do ReduceScatter with scale
+        VLOG(1) << "FP32 HasNanInf before all reduce: "
+                << HasNanInf(dev_ctx, fp32_grad, fp32_numel);
+        VLOG(1) << "FP16 HasNanInf before all reduce: "
+                << HasNanInf(dev_ctx, fp16_grad, fp16_numel);
         if (local_shard) {
           if (use_hierarchical_allreduce) {
             NCCLReduceScatterWithScale(

              << " , " << GetMinMaxStr(tensors[i]);
    }
  }
}

static void CheckHasNanInfGrad(const float *fp32_grad,

                                     stream,
                                     dev_ctx,
                                     fp16_scale);
        }
        VLOG(1) << "FP32 HasNanInf after all reduce: "
                << HasNanInf(dev_ctx, fp32_sum_grad, fp32_numel_each_device);
        VLOG(1) << "FP16 HasNanInf after all reduce: "
                << HasNanInf(dev_ctx, fp16_sum_grad, fp16_numel_each_device);
        CheckHasNanInfGrad(fp32_sum_grad,
                           fp32_numel_each_device,
                           fp16_sum_grad,
                           fp16_numel_each_device,
                           fp32_square_grad_norm,
                           stream,
                           &cub_tmp_buffer);
        if (num_devices > 1) {
          PADDLE_ENFORCE_GPU_SUCCESS(
              platform::dynload::ncclAllReduce(fp32_square_grad_norm,
                                               fp32_square_grad_norm,
                                               1,
                                               ncclFloat32,
                                               ncclSum,
                                               local_comm,
                                               stream));
          VLOG(1) << "Grad square norm after all reduce: "
                  << FlattenToString(fp32_square_grad_norm, 1, place);
        }
        // (4) mark max_global_grad_norm as 0, meaning that clip has been
        // already performed
