Stage 3 training crashing AttributeError: 'NoneType' object has no attribute 'shape' #23
Description
Stage 3 training crashes in the tm2t evaluator:
Moviepy - Done !
Moviepy - video ready <PROJECT_ROOT>/experiments/MoT_vae_stage3_2025-10-03-11-41-31/validate_motion/epoch_0/M005328_0.000000_1.000000.mp4
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "<PROJECT_ROOT>/train.py", line 97, in <module>
main()
File "<PROJECT_ROOT>/train.py", line 88, in main
trainer.fit(model, datamodule=datamodule)
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/trainer.py", line 520, in fit
call._call_and_handle_interrupt(
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt
return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
File "<ENV_PATH>/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 92, in launch
return function(*args, **kwargs)
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/trainer.py", line 559, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/trainer.py", line 935, in _run
results = self._run_stage()
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/trainer.py", line 976, in _run_stage
self._run_sanity_check()
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/trainer.py", line 1005, in _run_sanity_check
val_loop.run()
File "<ENV_PATH>/site-packages/pytorch_lightning/loops/utilities.py", line 174, in _decorator
return loop_run(self, *args, **kwargs)
File "<ENV_PATH>/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 115, in run
self._evaluation_step(batch, batch_idx, dataloader_idx)
File "<ENV_PATH>/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 375, in _evaluation_step
output = call._call_strategy_hook(trainer, hook_name, *step_kwargs.values())
File "<ENV_PATH>/site-packages/pytorch_lightning/trainer/call.py", line 288, in _call_strategy_hook
output = fn(*args, **kwargs)
File "<ENV_PATH>/site-packages/pytorch_lightning/strategies/ddp.py", line 337, in validation_step
return self.model(*args, **kwargs)
File "<ENV_PATH>/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "<ENV_PATH>/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward
output = self._run_ddp_forward(*inputs, **kwargs)
File "<ENV_PATH>/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward
return module_to_run(*inputs[0], **kwargs[0]) # type: ignore[index]
File "<ENV_PATH>/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "<ENV_PATH>/site-packages/pytorch_lightning/overrides/base.py", line 102, in forward
return self._forward_module.validation_step(*inputs, **kwargs)
File "<PROJECT_ROOT>/motGPT/models/base.py", line 34, in validation_step
loss = self.allsplit_step("val", batch, batch_idx)
File "<PROJECT_ROOT>/motGPT/models/motgpt.py", line 502, in allsplit_step
getattr(self.metrics, metric).update(
File "<ENV_PATH>/site-packages/torchmetrics/metric.py", line 263, in wrapped_func
return update(*args, **kwargs)
File "<ENV_PATH>/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "<PROJECT_ROOT>/motGPT/metrics/t2m.py", line 274, in update
text_emb = self.t2m_textencoder(word_embs, pos_ohot, text_lengths)
File "<ENV_PATH>/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "<PROJECT_ROOT>/motGPT/archs/tm2t_evaluator.py", line 137, in forward
num_samples = word_embs.shape[0]
AttributeError: 'NoneType' object has no attribute 'shape'