pipe/_exec_backward_pass: fix immediate grad update (#5605)

dont call update_hp_grads in _exec_backward_pass incase of immediate
grad update, as it would be called as part of bwd hooks

Author: nelyahu

deepspeed/runtime/pipe/engine.py

@@ -852,7 +852,8 @@ def _exec_backward_pass(self, buffer_id):
 
         if self.using_bf16_optimizer and not self.is_last_stage():
             # manually call because we don't call optimizer.backward()
-            self.optimizer.update_hp_grads(clear_lp_grads=False)
+            if not self._config.bfloat16_immediate_grad_update:
+                self.optimizer.update_hp_grads(clear_lp_grads=False)
 
         # Free up the memory from the output of forward()
         self.pipe_buffers['output_tensors'][buffer_id] = None