@pytest.mark.processor("gpu")
@pytest.mark.integration("smmp")
@pytest.mark.model("hf_qa_smmp")
@pytest.mark.skip_cpu
@pytest.mark.skip_py2_containers
@pytest.mark.skip_trcomp_containers
def test_smmp_gpu(
    ecr_image, sagemaker_regions, instance_type, framework_version, py_version, dist_gpu_backend
):

    def _check_job_status(self, job, desc, status_key_name):
        """Check to see if the job completed successfully.

        If not, construct and raise a exceptions. (UnexpectedStatusException).

        Args:
            job (str): The name of the job to check.
            desc (dict[str, str]): The result of ``describe_training_job()``.
            status_key_name (str): Status key name to check for.

        Raises:
            exceptions.CapacityError: If the training job fails with CapacityError.
            exceptions.UnexpectedStatusException: If the training job fails.
        """
        status = desc[status_key_name]
        # If the status is capital case, then convert it to Camel case
        status = _STATUS_CODE_TABLE.get(status, status)

        if status == "Stopped":
            LOGGER.warning(
                "Job ended with status 'Stopped' rather than 'Completed'. "
                "This could mean the job timed out or stopped early for some other reason: "
                "Consider checking whether it completed as you expect."
            )
        elif status != "Completed":
            reason = desc.get("FailureReason", "(No reason provided)")
            job_type = status_key_name.replace("JobStatus", " job")
            message = "Error for {job_type} {job_name}: {status}. Reason: {reason}".format(
                job_type=job_type, job_name=job, status=status, reason=reason
            )
            if "CapacityError" in str(reason):
                raise exceptions.CapacityError(
                    message=message,
                    allowed_statuses=["Completed", "Stopped"],
                    actual_status=status,
                )
>           raise exceptions.UnexpectedStatusException(
                message=message,
                allowed_statuses=["Completed", "Stopped"],
                actual_status=status,
            )
E           sagemaker.exceptions.UnexpectedStatusException: Error for Training job test-hf-pt-qa-smmp-1684001017-f81e: Failed. Reason: AlgorithmError: ExecuteUserScriptError:
E           ExitCode 1
E           ErrorMessage "AttributeError: 'TrainingArguments' object has no attribute 'distributed_state'
E            ╭───────────────────── Traceback (most recent call last) ──────────────────────╮
E            │ /opt/conda/lib/python3.10/runpy.py:196 in _run_module_as_main                │
E            │                                                                              │
E            │   193 │   main_globals = sys.modules["__main__"].__dict__                    │
E            │   194 │   if alter_argv:                                                     │
E            │   195 │   │   sys.argv[0] = mod_spec.origin                                  │
E            │ ❱ 196 │   return _run_code(code, main_globals, None,                         │
E            │   197 │   │   │   │   │    "__main__", mod_spec)                             │
E            │   198                                                                        │
E            │   199 de, exit code: 1

2.0.0-transformers4.29.1-gpu-py310-cu118-ubuntu20.04-pr-2993-2023-05-13-17-25-48/lib/python3.8/site-packages/sagemaker/session.py:3749: UnexpectedStatusException