Skip to content

Commit 1528732

Browse files
cmikeh2JamesTheZ
authored andcommitted
Workspace for Inference Kernels (#1)
* Add workspace capability to DSKernel * Add to injection pipeline * Validated
1 parent beed9c2 commit 1528732

4 files changed

Lines changed: 87 additions & 2 deletions

File tree

‎deepspeed/inference/v2/kernels/ds_kernel.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,17 @@
44
# DeepSpeed Team
55

66
from abc import ABC, abstractmethod
7+
from typing import Optional
8+
9+
import torch
10+
11+
from deepspeed.inference.v2.logging import inference_logger
712

813

914
class DSKernelBase(ABC):
1015

16+
_workspace: Optional[torch.Tensor] = None
17+
1118
@abstractmethod
1219
def __init__(self, *args, **kwargs):
1320
"""
@@ -30,3 +37,50 @@ def __call__(self, *args, **kwargs):
3037
should be performed here.
3138
"""
3239
raise NotImplementedError()
40+
41+
def requested_workspace_size(self) -> int:
42+
"""
43+
Return the requested workspace size in bytes.
44+
45+
This should be overloaded if the kernel requires a workspace.
46+
47+
Returns:
48+
int: Number of bytes necessary.
49+
"""
50+
return 0
51+
52+
def get_workspace(self, bytes: int) -> torch.Tensor:
53+
"""
54+
Return the data pointer to the scratchpad memory.
55+
56+
Args:
57+
bytes (int): Number of bytes necessary.
58+
59+
Raises:
60+
RuntimeError: If the workspace is not allocated.
61+
ValueError: If the workspace is not large enough.
62+
"""
63+
if DSKernelBase._workspace is None:
64+
raise RuntimeError("Workspace not allocated")
65+
if DSKernelBase._workspace.numel() < bytes:
66+
raise ValueError("Workspace too small")
67+
return DSKernelBase._workspace
68+
69+
@staticmethod
70+
def create_workspace(bytes: int) -> int:
71+
"""
72+
Create a workspace of the requested size.
73+
74+
Args:
75+
bytes (int): Number of bytes necessary.
76+
77+
Raises:
78+
RuntimeError: If the workspace is already allocated.
79+
"""
80+
if DSKernelBase._workspace is not None:
81+
raise RuntimeError("Workspace already allocated")
82+
83+
if bytes > 0:
84+
inference_logger().info(f"Allocating {bytes} bytes of workspace")
85+
86+
DSKernelBase._workspace = torch.empty((bytes, ), dtype=torch.uint8)

‎deepspeed/inference/v2/model_implementations/inference_model_base.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import torch
1010

1111
import deepspeed.comm as dist
12+
from deepspeed.inference.v2.kernels.ds_kernel import DSKernelBase
13+
from deepspeed.inference.v2.modules.ds_module import DSModuleBase
1214
from ..ragged import DSStateManager, RaggedBatchWrapper
1315
from ..ragged.manager_configs import KVCacheConfig
1416
from ..ragged import DSSequenceDescriptor
@@ -266,3 +268,19 @@ def forward(wrapped_batch: RaggedBatchWrapper) -> torch.Tensor:
266268
should not rely on the ability to use python control flow.
267269
"""
268270
raise NotImplementedError()
271+
272+
def initialize_kernel_workspace(self) -> None:
273+
"""
274+
Iterates over all kernels in the model and collects the requested workspace size. This
275+
workspace is then allocated and stored in the kernel base class.
276+
"""
277+
max_workspace_size = 0
278+
for module in self.modules():
279+
if not isinstance(module, DSModuleBase):
280+
continue
281+
282+
for kernel in module.kernels():
283+
max_workspace_size = max(max_workspace_size, kernel.requested_workspace_size())
284+
285+
if max_workspace_size > 0:
286+
DSKernelBase.create_workspace(max_workspace_size)

‎deepspeed/inference/v2/model_implementations/inference_policy_base.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,8 @@ def build_model(self, engine_config: RaggedInferenceEngineConfig, mp_group: Any)
153153
DSInferenceModelBase: An implementation of the inference model abstraction that will be
154154
run by the engine.
155155
"""
156-
self.model = self.instantiate_model(engine_config, mp_group)
156+
self.model: DSInferenceModelBase = self.instantiate_model(engine_config, mp_group)
157+
self.model.initialize_kernel_workspace()
157158
self.populate_model_parameters()
158159
return self.model
159160

‎deepspeed/inference/v2/modules/ds_module.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44
# DeepSpeed Team
55

66
from abc import ABC, abstractstaticmethod
7-
from typing import Any, Dict, Type
7+
from typing import Any, Dict, Iterable, Type
88

99
import torch
1010

1111
from deepspeed.runtime.config_utils import DeepSpeedConfigModel
12+
from deepspeed.inference.v2.kernels import DSKernelBase
1213

1314

1415
class DSModuleConfig(DeepSpeedConfigModel):
@@ -60,3 +61,14 @@ def __init__(self, config: DSModuleConfig, implementation_config: Dict[str, Any]
6061
super().__init__()
6162
self._config = config
6263
self._implementation_config = implementation_config
64+
65+
def kernels(self) -> Iterable[DSKernelBase]:
66+
"""
67+
Return an iterable of all kernels used by this module.
68+
69+
This should be implemented by the children of functionality modules and should report
70+
all kernels used by this module.
71+
"""
72+
for attr in dir(self):
73+
if isinstance(getattr(self, attr), DSKernelBase):
74+
yield getattr(self, attr)

0 commit comments

Comments
 (0)