-
Notifications
You must be signed in to change notification settings - Fork 781
nightly test error: unet_segmentation_3d_catalyst.ipynb #223
Copy link
Copy link
Closed
Description
Describe the bug
from log: https://github.com/Project-MONAI/MONAI/runs/2751603451?check_suite_focus=true
Running ./3d_segmentation/unet_segmentation_3d_catalyst.ipynb
Checking PEP8 compliance...
Running notebook...
Before:
"max_epochs = 600\n",
After:
"max_epochs = 1\n",
Before:
"val_interval = 2\n",
After:
"val_interval = 1\n",
Executing: 0%| | 0/27 [00:00<?, ?cell/s]
Executing: 4%|▎ | 1/27 [00:00<00:25, 1.02cell/s]
Executing: 11%|█ | 3/27 [00:18<01:18, 3.27s/cell]
Executing: 19%|█▊ | 5/27 [00:20<00:58, 2.68s/cell]
Executing: 44%|████▍ | 12/27 [00:26<00:31, 2.12s/cell]
Executing: 52%|█████▏ | 14/27 [00:27<00:20, 1.60s/cell]
Executing: 78%|███████▊ | 21/27 [00:30<00:07, 1.26s/cell]
Executing: 78%|███████▊ | 21/27 [00:32<00:09, 1.53s/cell]
Traceback (most recent call last):
File "/opt/conda/bin/papermill", line 8, in <module>
sys.exit(papermill())
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/opt/conda/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/papermill/cli.py", line 250, in papermill
execute_notebook(
File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 122, in execute_notebook
raise_for_execution_errors(nb, output_path)
File "/opt/conda/lib/python3.8/site-packages/papermill/execute.py", line 234, in raise_for_execution_errors
raise error
papermill.exceptions.PapermillExecutionError:
---------------------------------------------------------------------------
Exception encountered at "In [9]":
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-9-9674d37922eb> in <module>
5 input_key="img", input_target_key="seg", output_key="logits"
6 ) # you can also specify `device` here
----> 7 runner.train(
8 loaders={"train": train_loader, "valid": val_loader},
9 model=model,
/opt/conda/lib/python3.8/site-packages/catalyst/dl/runner/runner.py in train(self, model, criterion, optimizer, scheduler, datasets, loaders, callbacks, logdir, resume, num_epochs, valid_loader, main_metric, minimize_metric, verbose, stage_kwargs, checkpoint_data, fp16, distributed, check, overfit, timeit, load_best_on_end, initial_seed, state_kwargs)
161 )
162 self.experiment = experiment
--> 163 utils.distributed_cmd_run(self.run_experiment, distributed)
164
165 def infer(
/opt/conda/lib/python3.8/site-packages/catalyst/utils/scripts.py in distributed_cmd_run(worker_fn, distributed, *args, **kwargs)
130 or world_size <= 1
131 ):
--> 132 worker_fn(*args, **kwargs)
133 elif local_rank is not None:
134 torch.cuda.set_device(int(local_rank))
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in run_experiment(self, experiment)
985 if _exception_handler_check(getattr(self, "callbacks", None)):
986 self.exception = ex
--> 987 self._run_event("on_exception")
988 else:
989 raise ex
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_event(self, event)
778 """
779 for callback in self.callbacks.values():
--> 780 getattr(callback, event)(self)
781
782 def _batch2device(
/opt/conda/lib/python3.8/site-packages/catalyst/core/callbacks/exception.py in on_exception(self, runner)
22
23 if runner.need_exception_reraise:
---> 24 raise exception
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in run_experiment(self, experiment)
973 try:
974 for stage in self.experiment.stages:
--> 975 self._run_stage(stage)
976 except (Exception, KeyboardInterrupt) as ex:
977 from catalyst.core.callbacks.exception import ExceptionCallback
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_stage(self, stage)
941 )
942 self._run_event("on_epoch_start")
--> 943 self._run_epoch(stage=stage, epoch=self.epoch)
944 self._run_event("on_epoch_end")
945
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_epoch(self, stage, epoch)
920 self._run_event("on_loader_start")
921 with torch.set_grad_enabled(self.is_train_loader):
--> 922 self._run_loader(loader)
923 self._run_event("on_loader_end")
924
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_loader(self, loader)
855 self.global_batch_step += 1
856 self.loader_batch_step = i + 1
--> 857 self._run_batch(batch)
858 if self.need_early_stop:
859 self.need_early_stop = False
/opt/conda/lib/python3.8/site-packages/catalyst/core/runner.py in _run_batch(self, batch)
820 """
821 if isinstance(batch, dict):
--> 822 self.batch_size = next(iter(batch.values())).shape[0]
823 else:
824 self.batch_size = len(batch[0])
AttributeError: 'dict' object has no attribute 'shape'
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
No labels