Skip to content

nightly test error -- ./modules/mednist_GAN_workflow_array.ipynb #349

@wyli

Description

@wyli

Describe the bug
from the nightly build log of 12th/sept

Running ./modules/mednist_GAN_workflow_array.ipynb
Checking PEP8 compliance...
Running notebook...
Before:
    "max_epochs = 50\n",
After:
    "max_epochs = 1\n",
Before:
    "disc_train_steps = 5\n",
After:
    "disc_train_steps = 1\n",

Executing:   0%|          | 0/33 [00:00<?, ?cell/s]
Executing:   3%|▎         | 1/33 [00:00<00:31,  1.01cell/s]
Executing:   9%|▉         | 3/33 [00:04<00:37,  1.27s/cell]
Executing:  12%|█▏        | 4/33 [00:06<00:39,  1.36s/cell]
Executing:  30%|███       | 10/33 [00:16<00:33,  1.48s/cell]
Executing:  52%|█████▏    | 17/33 [00:25<00:22,  1.42s/cell]
Executing:  61%|██████    | 20/33 [00:27<00:15,  1.18s/cell]
Executing:  79%|███████▉  | 26/33 [00:30<00:06,  1.02cell/s]
Executing:  79%|███████▉  | 26/33 [00:32<00:08,  1.24s/cell]
Traceback (most recent call last):
  File "/opt/conda/bin/papermill", line 8, in <module>
    sys.exit(papermill())
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/papermill/cli.py", line 250, in papermill
    execute_notebook(
  File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 122, in execute_notebook
    raise_for_execution_errors(nb, output_path)
  File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 234, in raise_for_execution_errors
    raise error
papermill.exceptions.PapermillExecutionError: 
---------------------------------------------------------------------------
Exception encountered at "In [14]":
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-14-041e2033e90a> in <module>
----> 1 trainer.run()

/opt/conda/lib/python3.8/site-packages/monai/engines/trainer.py in run(self)
     54         """
     55         self.scaler = torch.cuda.amp.GradScaler() if self.amp else None
---> 56         super().run()
     57 
     58     def get_train_stats(self) -> Dict[str, float]:

/opt/conda/lib/python3.8/site-packages/monai/engines/workflow.py in run(self)
    248 
    249         """
--> 250         super().run(data=self.data_loader, max_epochs=self.state.max_epochs)
    251 
    252     def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]):

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in run(self, data, max_epochs, epoch_length, seed)
    699 
    700         self.state.dataloader = data
--> 701         return self._internal_run()
    702 
    703     @staticmethod

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _internal_run(self)
    772             self._dataloader_iter = None
    773             self.logger.error(f"Engine run is terminating due to exception: {e}")
--> 774             self._handle_exception(e)
    775 
    776         self._dataloader_iter = None

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _handle_exception(self, e)
    465     def _handle_exception(self, e: BaseException) -> None:
    466         if Events.EXCEPTION_RAISED in self._event_handlers:
--> 467             self._fire_event(Events.EXCEPTION_RAISED, e)
    468         else:
    469             raise e

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _fire_event(self, event_name, *event_args, **event_kwargs)
    422             kwargs.update(event_kwargs)
    423             first, others = ((args[0],), args[1:]) if (args and args[0] == self) else ((), args)
--> 424             func(*first, *(event_args + others), **kwargs)
    425 
    426     def fire_event(self, event_name: Any) -> None:

/opt/conda/lib/python3.8/site-packages/monai/handlers/stats_handler.py in exception_raised(self, engine, e)
    146         """
    147         self.logger.exception(f"Exception: {e}")
--> 148         raise e
    149 
    150     def _default_epoch_print(self, engine: Engine) -> None:

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _internal_run(self)
    742                     self._setup_engine()
    743 
--> 744                 time_taken = self._run_once_on_dataset()
    745                 # time is available for handlers but must be update after fire
    746                 self.state.times[Events.EPOCH_COMPLETED.name] = time_taken

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _run_once_on_dataset(self)
    846         except Exception as e:
    847             self.logger.error(f"Current run is terminating due to exception: {e}")
--> 848             self._handle_exception(e)
    849 
    850         return time.time() - start_time

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _handle_exception(self, e)
    465     def _handle_exception(self, e: BaseException) -> None:
    466         if Events.EXCEPTION_RAISED in self._event_handlers:
--> 467             self._fire_event(Events.EXCEPTION_RAISED, e)
    468         else:
    469             raise e

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _fire_event(self, event_name, *event_args, **event_kwargs)
    422             kwargs.update(event_kwargs)
    423             first, others = ((args[0],), args[1:]) if (args and args[0] == self) else ((), args)
--> 424             func(*first, *(event_args + others), **kwargs)
    425 
    426     def fire_event(self, event_name: Any) -> None:

/opt/conda/lib/python3.8/site-packages/monai/handlers/stats_handler.py in exception_raised(self, engine, e)
    146         """
    147         self.logger.exception(f"Exception: {e}")
--> 148         raise e
    149 
    150     def _default_epoch_print(self, engine: Engine) -> None:

/opt/conda/lib/python3.8/site-packages/ignite/engine/engine.py in _run_once_on_dataset(self)
    832                 self.state.iteration += 1
    833                 self._fire_event(Events.ITERATION_STARTED)
--> 834                 self.state.output = self._process_function(self, self.state.batch)
    835                 self._fire_event(Events.ITERATION_COMPLETED)
    836 

/opt/conda/lib/python3.8/site-packages/monai/engines/trainer.py in _iteration(self, engine, batchdata)
    361             else:
    362                 self.d_optimizer.zero_grad(set_to_none=self.optim_set_to_none)
--> 363             dloss = self.d_loss_function(g_output, d_input)
    364             dloss.backward()
    365             self.d_optimizer.step()

<ipython-input-11-aa4677288b64> in discriminator_loss(gen_images, real_images)
     39     gen = gen_images.new_full((gen_images.shape[0], 1), fake_label)
     40 
---> 41     realloss = disc_loss_criterion(disc_net(real_images), real)
     42     genloss = disc_loss_criterion(disc_net(gen_images.detach()), gen)
     43 

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1013         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1014                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015             return forward_call(*input, **kwargs)
   1016         # Do not call functions when jit is used
   1017         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.8/site-packages/monai/networks/nets/regressor.py in forward(self, x)
    139 
    140     def forward(self, x: torch.Tensor) -> torch.Tensor:
--> 141         x = self.net(x)
    142         x = self.final(x)
    143         x = self.reshape(x)

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1013         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1014                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015             return forward_call(*input, **kwargs)
   1016         # Do not call functions when jit is used
   1017         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py in forward(self, input)
    116     def forward(self, input):
    117         for module in self:
--> 118             input = module(input)
    119         return input
    120 

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1013         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1014                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015             return forward_call(*input, **kwargs)
   1016         # Do not call functions when jit is used
   1017         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.8/site-packages/monai/networks/blocks/convolutions.py in forward(self, x)
    322 
    323     def forward(self, x: torch.Tensor) -> torch.Tensor:
--> 324         res: torch.Tensor = self.residual(x)  # create the additive residual from x
    325         cx: torch.Tensor = self.conv(x)  # apply x to sequence of operations
    326         return cx + res  # add the residual to the output

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1013         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1014                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1015             return forward_call(*input, **kwargs)
   1016         # Do not call functions when jit is used
   1017         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py in forward(self, input)
    397 
    398     def forward(self, input: Tensor) -> Tensor:
--> 399         return self._conv_forward(input, self.weight, self.bias)
    400 
    401 class Conv3d(_ConvNd):

/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias)
    393                             weight, bias, self.stride,
    394                             _pair(0), self.dilation, self.groups)
--> 395         return F.conv2d(input, weight, bias, self.stride,
    396                         self.padding, self.dilation, self.groups)
    397 

RuntimeError: Input type (torch.cuda.DoubleTensor) and weight type (torch.cuda.FloatTensor) should be the same

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions