-
Notifications
You must be signed in to change notification settings - Fork 781
nightly test error -- ./modules/mednist_GAN_workflow_array.ipynb #349
Copy link
Copy link
Closed
Project-MONAI/MONAI
#2941Description
Describe the bug
from the nightly build log of 12th/sept
Running ./modules/mednist_GAN_workflow_array.ipynb
Checking PEP8 compliance...
Running notebook...
Before:
"max_epochs = 50\n",
After:
"max_epochs = 1\n",
Before:
"disc_train_steps = 5\n",
After:
"disc_train_steps = 1\n",
Executing: 0%| | 0/33 [00:00<?, ?cell/s]
Executing: 3%|▎ | 1/33 [00:00<00:31, 1.01cell/s]
Executing: 9%|▉ | 3/33 [00:04<00:37, 1.27s/cell]
Executing: 12%|█▏ | 4/33 [00:06<00:39, 1.36s/cell]
Executing: 30%|███ | 10/33 [00:16<00:33, 1.48s/cell]
Executing: 52%|█████▏ | 17/33 [00:25<00:22, 1.42s/cell]
Executing: 61%|██████ | 20/33 [00:27<00:15, 1.18s/cell]
Executing: 79%|███████▉ | 26/33 [00:30<00:06, 1.02cell/s]
Executing: 79%|███████▉ | 26/33 [00:32<00:08, 1.24s/cell]
Traceback (most recent call last):
File "/opt/conda/bin/papermill", line 8, in <module>
sys.exit(papermill())
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/papermill/cli.py", line 250, in papermill
execute_notebook(
File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 122, in execute_notebook
raise_for_execution_errors(nb, output_path)
File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 234, in raise_for_execution_errors
raise error
papermill.exceptions.PapermillExecutionError:
---------------------------------------------------------------------------
Exception encountered at "In [14]":
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-14-041e2033e90a> in <module>
----> 1 trainer.run()
/opt/conda/lib/python3.8/site-packages/monai/engines/trainer.py in run(self)
54 """
55 self.scaler = torch.cuda.amp.GradScaler() if self.amp else None
---> 56 super().run()
57
58 def get_train_stats(self) -> Dict[str, float]:
/opt/conda/lib/python3.8/site-packages/monai/engines/workflow.py in run(self)
248
249 """
--> 250 super().run(data=self.data_loader, max_epochs=self.state.max_epochs)
251
252 def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]):
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in run(self, data, max_epochs, epoch_length, seed)
699
700 self.state.dataloader = data
--> 701 return self._internal_run()
702
703 @staticmethod
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _internal_run(self)
772 self._dataloader_iter = None
773 self.logger.error(f"Engine run is terminating due to exception: {e}")
--> 774 self._handle_exception(e)
775
776 self._dataloader_iter = None
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _handle_exception(self, e)
465 def _handle_exception(self, e: BaseException) -> None:
466 if Events.EXCEPTION_RAISED in self._event_handlers:
--> 467 self._fire_event(Events.EXCEPTION_RAISED, e)
468 else:
469 raise e
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _fire_event(self, event_name, *event_args, **event_kwargs)
422 kwargs.update(event_kwargs)
423 first, others = ((args[0],), args[1:]) if (args and args[0] == self) else ((), args)
--> 424 func(*first, *(event_args + others), **kwargs)
425
426 def fire_event(self, event_name: Any) -> None:
/opt/conda/lib/python3.8/site-packages/monai/handlers/stats_handler.py in exception_raised(self, engine, e)
146 """
147 self.logger.exception(f"Exception: {e}")
--> 148 raise e
149
150 def _default_epoch_print(self, engine: Engine) -> None:
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _internal_run(self)
742 self._setup_engine()
743
--> 744 time_taken = self._run_once_on_dataset()
745 # time is available for handlers but must be update after fire
746 self.state.times[Events.EPOCH_COMPLETED.name] = time_taken
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _run_once_on_dataset(self)
846 except Exception as e:
847 self.logger.error(f"Current run is terminating due to exception: {e}")
--> 848 self._handle_exception(e)
849
850 return time.time() - start_time
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _handle_exception(self, e)
465 def _handle_exception(self, e: BaseException) -> None:
466 if Events.EXCEPTION_RAISED in self._event_handlers:
--> 467 self._fire_event(Events.EXCEPTION_RAISED, e)
468 else:
469 raise e
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _fire_event(self, event_name, *event_args, **event_kwargs)
422 kwargs.update(event_kwargs)
423 first, others = ((args[0],), args[1:]) if (args and args[0] == self) else ((), args)
--> 424 func(*first, *(event_args + others), **kwargs)
425
426 def fire_event(self, event_name: Any) -> None:
/opt/conda/lib/python3.8/site-packages/monai/handlers/stats_handler.py in exception_raised(self, engine, e)
146 """
147 self.logger.exception(f"Exception: {e}")
--> 148 raise e
149
150 def _default_epoch_print(self, engine: Engine) -> None:
/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _run_once_on_dataset(self)
832 self.state.iteration += 1
833 self._fire_event(Events.ITERATION_STARTED)
--> 834 self.state.output = self._process_function(self, self.state.batch)
835 self._fire_event(Events.ITERATION_COMPLETED)
836
/opt/conda/lib/python3.8/site-packages/monai/engines/trainer.py in _iteration(self, engine, batchdata)
361 else:
362 self.d_optimizer.zero_grad(set_to_none=self.optim_set_to_none)
--> 363 dloss = self.d_loss_function(g_output, d_input)
364 dloss.backward()
365 self.d_optimizer.step()
<ipython-input-11-aa4677288b64> in discriminator_loss(gen_images, real_images)
39 gen = gen_images.new_full((gen_images.shape[0], 1), fake_label)
40
---> 41 realloss = disc_loss_criterion(disc_net(real_images), real)
42 genloss = disc_loss_criterion(disc_net(gen_images.detach()), gen)
43
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1013 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1014 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015 return forward_call(*input, **kwargs)
1016 # Do not call functions when jit is used
1017 full_backward_hooks, non_full_backward_hooks = [], []
/opt/conda/lib/python3.8/site-packages/monai/networks/nets/regressor.py in forward(self, x)
139
140 def forward(self, x: torch.Tensor) -> torch.Tensor:
--> 141 x = self.net(x)
142 x = self.final(x)
143 x = self.reshape(x)
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1013 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1014 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015 return forward_call(*input, **kwargs)
1016 # Do not call functions when jit is used
1017 full_backward_hooks, non_full_backward_hooks = [], []
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py in forward(self, input)
116 def forward(self, input):
117 for module in self:
--> 118 input = module(input)
119 return input
120
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1013 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1014 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015 return forward_call(*input, **kwargs)
1016 # Do not call functions when jit is used
1017 full_backward_hooks, non_full_backward_hooks = [], []
/opt/conda/lib/python3.8/site-packages/monai/networks/blocks/convolutions.py in forward(self, x)
322
323 def forward(self, x: torch.Tensor) -> torch.Tensor:
--> 324 res: torch.Tensor = self.residual(x) # create the additive residual from x
325 cx: torch.Tensor = self.conv(x) # apply x to sequence of operations
326 return cx + res # add the residual to the output
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1013 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1014 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015 return forward_call(*input, **kwargs)
1016 # Do not call functions when jit is used
1017 full_backward_hooks, non_full_backward_hooks = [], []
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py in forward(self, input)
397
398 def forward(self, input: Tensor) -> Tensor:
--> 399 return self._conv_forward(input, self.weight, self.bias)
400
401 class Conv3d(_ConvNd):
/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias)
393 weight, bias, self.stride,
394 _pair(0), self.dilation, self.groups)
--> 395 return F.conv2d(input, weight, bias, self.stride,
396 self.padding, self.dilation, self.groups)
397
RuntimeError: Input type (torch.cuda.DoubleTensor) and weight type (torch.cuda.FloatTensor) should be the same
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
No labels