Skip to content

nightly test error: unet_segmentation_3d_catalyst.ipynb #223

@wyli

Description

@wyli

Describe the bug
from log: https://github.com/Project-MONAI/MONAI/runs/2751603451?check_suite_focus=true

Running ./3d_segmentation/unet_segmentation_3d_catalyst.ipynb
Checking PEP8 compliance...
Running notebook...
Before:
    "max_epochs = 600\n",
After:
    "max_epochs = 1\n",
Before:
    "val_interval = 2\n",
After:
    "val_interval = 1\n",

Executing:   0%|          | 0/27 [00:00<?, ?cell/s]
Executing:   4%|▎         | 1/27 [00:00<00:25,  1.02cell/s]
Executing:  11%|█         | 3/27 [00:18<01:18,  3.27s/cell]
Executing:  19%|█▊        | 5/27 [00:20<00:58,  2.68s/cell]
Executing:  44%|████▍     | 12/27 [00:26<00:31,  2.12s/cell]
Executing:  52%|█████▏    | 14/27 [00:27<00:20,  1.60s/cell]
Executing:  78%|███████▊  | 21/27 [00:30<00:07,  1.26s/cell]
Executing:  78%|███████▊  | 21/27 [00:32<00:09,  1.53s/cell]
Traceback (most recent call last):
  File "/opt/conda/bin/papermill", line 8, in <module>
    sys.exit(papermill())
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/papermill/cli.py", line 250, in papermill
    execute_notebook(
  File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 122, in execute_notebook
    raise_for_execution_errors(nb, output_path)
  File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 234, in raise_for_execution_errors
    raise error
papermill.exceptions.PapermillExecutionError: 
---------------------------------------------------------------------------
Exception encountered at "In [9]":
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-9-9674d37922eb> in <module>
      5     input_key="img", input_target_key="seg", output_key="logits"
      6 )  # you can also specify `device` here
----> 7 runner.train(
      8     loaders={"train": train_loader, "valid": val_loader},
      9     model=model,

/opt/conda/lib/python3.8/site-packages/catalyst/dl/runner/runner.py in train(self, model, criterion, optimizer, scheduler, datasets, loaders, callbacks, logdir, resume, num_epochs, valid_loader, main_metric, minimize_metric, verbose, stage_kwargs, checkpoint_data, fp16, distributed, check, overfit, timeit, load_best_on_end, initial_seed, state_kwargs)
    161         )
    162         self.experiment = experiment
--> 163         utils.distributed_cmd_run(self.run_experiment, distributed)
    164 
    165     def infer(

/opt/conda/lib/python3.8/site-packages/catalyst/utils/scripts.py in distributed_cmd_run(worker_fn, distributed, *args, **kwargs)
    130         or world_size <= 1
    131     ):
--> 132         worker_fn(*args, **kwargs)
    133     elif local_rank is not None:
    134         torch.cuda.set_device(int(local_rank))

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in run_experiment(self, experiment)
    985             if _exception_handler_check(getattr(self, "callbacks", None)):
    986                 self.exception = ex
--> 987                 self._run_event("on_exception")
    988             else:
    989                 raise ex

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_event(self, event)
    778         """
    779         for callback in self.callbacks.values():
--> 780             getattr(callback, event)(self)
    781 
    782     def _batch2device(

/opt/conda/lib/python3.8/site-packages/catalyst/core/callbacks/exception.py in on_exception(self, runner)
     22 
     23         if runner.need_exception_reraise:
---> 24             raise exception

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in run_experiment(self, experiment)
    973         try:
    974             for stage in self.experiment.stages:
--> 975                 self._run_stage(stage)
    976         except (Exception, KeyboardInterrupt) as ex:
    977             from catalyst.core.callbacks.exception import ExceptionCallback

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_stage(self, stage)
    941             )
    942             self._run_event("on_epoch_start")
--> 943             self._run_epoch(stage=stage, epoch=self.epoch)
    944             self._run_event("on_epoch_end")
    945 

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_epoch(self, stage, epoch)
    920             self._run_event("on_loader_start")
    921             with torch.set_grad_enabled(self.is_train_loader):
--> 922                 self._run_loader(loader)
    923             self._run_event("on_loader_end")
    924 

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_loader(self, loader)
    855             self.global_batch_step += 1
    856             self.loader_batch_step = i + 1
--> 857             self._run_batch(batch)
    858             if self.need_early_stop:
    859                 self.need_early_stop = False

/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_batch(self, batch)
    820         """
    821         if isinstance(batch, dict):
--> 822             self.batch_size = next(iter(batch.values())).shape[0]
    823         else:
    824             self.batch_size = len(batch[0])

AttributeError: 'dict' object has no attribute 'shape'


Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions