|
|
|
|
@ -170,6 +170,7 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
|
|
|
|
|
# logger.info([x.item() for x in losses] + [global_step, lr])
|
|
|
|
|
pbar_train_loader.set_postfix(loss=[round(y, 2) for y in [x.item() for x in losses]], lr=lr)
|
|
|
|
|
|
|
|
|
|
if global_step % hps.train.log_interval == 0:
|
|
|
|
|
scalar_dict = {
|
|
|
|
|
"loss/g/total": loss_gen_all,
|
|
|
|
|
"loss/d/total": loss_disc_all,
|
|
|
|
|
@ -187,9 +188,10 @@ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loade
|
|
|
|
|
"slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()),
|
|
|
|
|
"all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
|
|
|
|
|
}
|
|
|
|
|
global_step += 1
|
|
|
|
|
utils.summarize(writer=writer, global_step=global_step, images=image_dict, scalars=scalar_dict)
|
|
|
|
|
|
|
|
|
|
global_step += 1
|
|
|
|
|
|
|
|
|
|
if rank == 0:
|
|
|
|
|
# logger.info('====> Epoch: {}'.format(epoch))
|
|
|
|
|
|
|
|
|
|
|