Skip to content

Commit d36cc7c

Browse files
ambarveambarve
authored
Support for multiple SCSI controllers (#1328)
* Support for multiple SCSI controllers Enable using upto 4 SCSI controllers for LCOW UVMs. HCS currently doesn't respect the SCSI controller number provided with the Add SCSI disk requests. Hence, the SCSI disk can show up at some different controller inside the LCOW UVM. To avoid this, now we use GUIDs to represent each controller and use that GUID with the Add SCSI disk request. GCS code is also modified to identify the controller number from the controller GUID. Now if a LCOW pod is created with an annotation that sets VPMEM device count to 0, we will automatically enable 4 SCSI controllers. Even the rootfs.vhd will be attached via SCSI in that scenario. Signed-off-by: Amit Barve <[email protected]>
1 parent 6dd7225 commit d36cc7c

18 files changed

Lines changed: 460 additions & 178 deletions

File tree

‎cmd/runhcs/create-scratch.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ var createScratchCommand = cli.Command{
5252
// 256MB with boot from vhd supported.
5353
opts.MemorySizeInMB = 256
5454
opts.VPMemDeviceCount = 1
55+
// Default SCSI controller count is 4, we don't need that for this UVM,
56+
// bring it back to 1 to avoid any confusion with SCSI controller numbers.
57+
opts.SCSIControllerCount = 1
5558

5659
sizeGB := uint32(context.Uint("sizeGB"))
5760
if sizeGB == 0 {

‎cmd/runhcs/prepare-disk.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ var prepareDiskCommand = cli.Command{
4444
}
4545

4646
opts := uvm.NewDefaultOptionsLCOW("preparedisk-uvm", context.GlobalString("owner"))
47+
// Default SCSI controller count is 4, we don't need that for this UVM,
48+
// bring it back to 1 to avoid any confusion with SCSI controller numbers.
49+
opts.SCSIControllerCount = 1
4750

4851
preparediskUVM, err := uvm.CreateLCOW(ctx, opts)
4952
if err != nil {

‎internal/guest/storage/scsi/scsi.go‎

Lines changed: 96 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ import (
88
"fmt"
99
"io/ioutil"
1010
"os"
11+
"path"
1112
"path/filepath"
13+
"strconv"
14+
"strings"
1215
"time"
1316

1417
"github.com/pkg/errors"
@@ -20,6 +23,7 @@ import (
2023
dm "github.com/Microsoft/hcsshim/internal/guest/storage/devicemapper"
2124
"github.com/Microsoft/hcsshim/internal/log"
2225
"github.com/Microsoft/hcsshim/internal/oc"
26+
"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
2327
"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
2428
"github.com/Microsoft/hcsshim/pkg/securitypolicy"
2529
)
@@ -39,19 +43,53 @@ var (
3943
)
4044

4145
const (
42-
scsiDevicesPath = "/sys/bus/scsi/devices"
43-
verityDeviceFmt = "verity-scsi-contr%d-lun%d-%s"
46+
scsiDevicesPath = "/sys/bus/scsi/devices"
47+
vmbusDevicesPath = "/sys/bus/vmbus/devices"
48+
verityDeviceFmt = "verity-scsi-contr%d-lun%d-%s"
4449
)
4550

46-
// Mount creates a mount from the SCSI device on `controller` index `lun` to
51+
// fetchActualControllerNumber retrieves the actual controller number assigned to a SCSI controller
52+
// with number `passedController`.
53+
// When HCS creates the UVM it adds 4 SCSI controllers to the UVM but the 1st SCSI
54+
// controller according to HCS can actually show up as 2nd, 3rd or 4th controller inside
55+
// the UVM. So the i'th controller from HCS' perspective could actually be j'th controller
56+
// inside the UVM. However, we can refer to the SCSI controllers with their GUIDs (that
57+
// are hardcoded) and then using that GUID find out the SCSI controller number inside the
58+
// guest. This function does exactly that.
59+
func fetchActualControllerNumber(ctx context.Context, passedController uint8) (uint8, error) {
60+
// find the controller number by looking for a file named host<N> (e.g host1, host3 etc.)
61+
// `N` is the controller number.
62+
// Full file path would be /sys/bus/vmbus/devices/<controller-guid>/host<N>.
63+
controllerDirPath := path.Join(vmbusDevicesPath, guestrequest.ScsiControllerGuids[passedController])
64+
entries, err := ioutil.ReadDir(controllerDirPath)
65+
if err != nil {
66+
return 0, err
67+
}
68+
69+
for _, entry := range entries {
70+
baseName := path.Base(entry.Name())
71+
if !strings.HasPrefix(baseName, "host") {
72+
continue
73+
}
74+
controllerStr := baseName[len("host"):]
75+
controllerNum, err := strconv.ParseUint(controllerStr, 10, 8)
76+
if err != nil {
77+
return 0, fmt.Errorf("failed to parse controller number from %s: %w", baseName, err)
78+
}
79+
return uint8(controllerNum), nil
80+
}
81+
return 0, fmt.Errorf("host<N> directory not found inside %s", controllerDirPath)
82+
}
83+
84+
// mount creates a mount from the SCSI device on `controller` index `lun` to
4785
// `target`
4886
//
4987
// `target` will be created. On mount failure the created `target` will be
5088
// automatically cleaned up.
5189
//
5290
// If `encrypted` is set to true, the SCSI device will be encrypted using
5391
// dm-crypt.
54-
func Mount(
92+
func mount(
5593
ctx context.Context,
5694
controller,
5795
lun uint8,
@@ -159,10 +197,30 @@ func Mount(
159197
return nil
160198
}
161199

162-
// Unmount unmounts a SCSI device mounted at `target`.
200+
// Mount is just a wrapper over actual mount call. This wrapper finds out the controller
201+
// number from the controller GUID string and calls mount.
202+
func Mount(
203+
ctx context.Context,
204+
controller,
205+
lun uint8,
206+
target string,
207+
readonly bool,
208+
encrypted bool,
209+
options []string,
210+
verityInfo *guestresource.DeviceVerityInfo,
211+
securityPolicy securitypolicy.SecurityPolicyEnforcer,
212+
) (err error) {
213+
cNum, err := fetchActualControllerNumber(ctx, controller)
214+
if err != nil {
215+
return err
216+
}
217+
return mount(ctx, cNum, lun, target, readonly, encrypted, options, verityInfo, securityPolicy)
218+
}
219+
220+
// unmount unmounts a SCSI device mounted at `target`.
163221
//
164222
// If `encrypted` is true, it removes all its associated dm-crypto state.
165-
func Unmount(
223+
func unmount(
166224
ctx context.Context,
167225
controller,
168226
lun uint8,
@@ -206,6 +264,24 @@ func Unmount(
206264
return nil
207265
}
208266

267+
// Unmount is just a wrapper over actual unmount call. This wrapper finds out the controller
268+
// number from the controller GUID string and calls mount.
269+
func Unmount(
270+
ctx context.Context,
271+
controller,
272+
lun uint8,
273+
target string,
274+
encrypted bool,
275+
verityInfo *guestresource.DeviceVerityInfo,
276+
securityPolicy securitypolicy.SecurityPolicyEnforcer,
277+
) (err error) {
278+
cNum, err := fetchActualControllerNumber(ctx, controller)
279+
if err != nil {
280+
return err
281+
}
282+
return unmount(ctx, cNum, lun, target, encrypted, verityInfo, securityPolicy)
283+
}
284+
209285
// ControllerLunToName finds the `/dev/sd*` path to the SCSI device on
210286
// `controller` index `lun`.
211287
func ControllerLunToName(ctx context.Context, controller, lun uint8) (_ string, err error) {
@@ -217,8 +293,7 @@ func ControllerLunToName(ctx context.Context, controller, lun uint8) (_ string,
217293
trace.Int64Attribute("controller", int64(controller)),
218294
trace.Int64Attribute("lun", int64(lun)))
219295

220-
scsiID := fmt.Sprintf("0:0:%d:%d", controller, lun)
221-
296+
scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
222297
// Devices matching the given SCSI code should each have a subdirectory
223298
// under /sys/bus/scsi/devices/<scsiID>/block.
224299
blockPath := filepath.Join(scsiDevicesPath, scsiID, "block")
@@ -249,11 +324,11 @@ func ControllerLunToName(ctx context.Context, controller, lun uint8) (_ string,
249324
return devicePath, nil
250325
}
251326

252-
// UnplugDevice finds the SCSI device on `controller` index `lun` and issues a
327+
// unplugDevice finds the SCSI device on `controller` index `lun` and issues a
253328
// guest initiated unplug.
254329
//
255330
// If the device is not attached returns no error.
256-
func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
331+
func unplugDevice(ctx context.Context, controller, lun uint8) (err error) {
257332
_, span := trace.StartSpan(ctx, "scsi::UnplugDevice")
258333
defer span.End()
259334
defer func() { oc.SetSpanStatus(span, err) }()
@@ -262,7 +337,7 @@ func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
262337
trace.Int64Attribute("controller", int64(controller)),
263338
trace.Int64Attribute("lun", int64(lun)))
264339

265-
scsiID := fmt.Sprintf("0:0:%d:%d", controller, lun)
340+
scsiID := fmt.Sprintf("%d:0:0:%d", controller, lun)
266341
f, err := os.OpenFile(filepath.Join(scsiDevicesPath, scsiID, "delete"), os.O_WRONLY, 0644)
267342
if err != nil {
268343
if os.IsNotExist(err) {
@@ -277,3 +352,13 @@ func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
277352
}
278353
return nil
279354
}
355+
356+
// UnplugDevice is just a wrapper over actual unplugDevice call. This wrapper finds out the controller
357+
// number from the controller GUID string and calls unplugDevice.
358+
func UnplugDevice(ctx context.Context, controller, lun uint8) (err error) {
359+
cNum, err := fetchActualControllerNumber(ctx, controller)
360+
if err != nil {
361+
return err
362+
}
363+
return unplugDevice(ctx, cNum, lun)
364+
}

‎internal/guest/storage/scsi/scsi_test.go‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func Test_Mount_Mkdir_Fails_Error(t *testing.T) {
3737
return "", nil
3838
}
3939

40-
if err := Mount(
40+
if err := mount(
4141
context.Background(),
4242
0,
4343
0,
@@ -74,7 +74,7 @@ func Test_Mount_Mkdir_ExpectedPath(t *testing.T) {
7474
return nil
7575
}
7676

77-
if err := Mount(
77+
if err := mount(
7878
context.Background(),
7979
0,
8080
0,
@@ -111,7 +111,7 @@ func Test_Mount_Mkdir_ExpectedPerm(t *testing.T) {
111111
return nil
112112
}
113113

114-
if err := Mount(
114+
if err := mount(
115115
context.Background(),
116116
0,
117117
0,
@@ -148,7 +148,7 @@ func Test_Mount_ControllerLunToName_Valid_Controller(t *testing.T) {
148148
return nil
149149
}
150150

151-
if err := Mount(
151+
if err := mount(
152152
context.Background(),
153153
expectedController,
154154
0,
@@ -185,7 +185,7 @@ func Test_Mount_ControllerLunToName_Valid_Lun(t *testing.T) {
185185
return nil
186186
}
187187

188-
if err := Mount(
188+
if err := mount(
189189
context.Background(),
190190
0,
191191
expectedLun,
@@ -225,7 +225,7 @@ func Test_Mount_Calls_RemoveAll_OnMountFailure(t *testing.T) {
225225
return expectedErr
226226
}
227227

228-
if err := Mount(
228+
if err := mount(
229229
context.Background(),
230230
0,
231231
0,
@@ -263,7 +263,7 @@ func Test_Mount_Valid_Source(t *testing.T) {
263263
}
264264
return nil
265265
}
266-
err := Mount(context.Background(), 0, 0, "/fake/path", false, false, nil, nil, openDoorSecurityPolicyEnforcer())
266+
err := mount(context.Background(), 0, 0, "/fake/path", false, false, nil, nil, openDoorSecurityPolicyEnforcer())
267267
if err != nil {
268268
t.Fatalf("expected nil err, got: %v", err)
269269
}
@@ -290,7 +290,7 @@ func Test_Mount_Valid_Target(t *testing.T) {
290290
return nil
291291
}
292292

293-
if err := Mount(
293+
if err := mount(
294294
context.Background(),
295295
0,
296296
0,
@@ -326,7 +326,7 @@ func Test_Mount_Valid_FSType(t *testing.T) {
326326
return nil
327327
}
328328

329-
if err := Mount(
329+
if err := mount(
330330
context.Background(),
331331
0,
332332
0,
@@ -362,7 +362,7 @@ func Test_Mount_Valid_Flags(t *testing.T) {
362362
return nil
363363
}
364364

365-
if err := Mount(
365+
if err := mount(
366366
context.Background(),
367367
0,
368368
0,
@@ -398,7 +398,7 @@ func Test_Mount_Readonly_Valid_Flags(t *testing.T) {
398398
return nil
399399
}
400400

401-
if err := Mount(
401+
if err := mount(
402402
context.Background(),
403403
0,
404404
0,
@@ -433,7 +433,7 @@ func Test_Mount_Valid_Data(t *testing.T) {
433433
return nil
434434
}
435435

436-
if err := Mount(
436+
if err := mount(
437437
context.Background(),
438438
0,
439439
0,
@@ -469,7 +469,7 @@ func Test_Mount_Readonly_Valid_Data(t *testing.T) {
469469
return nil
470470
}
471471

472-
if err := Mount(
472+
if err := mount(
473473
context.Background(),
474474
0,
475475
0,
@@ -506,7 +506,7 @@ func Test_Read_Only_Security_Policy_Enforcement_Mount_Calls(t *testing.T) {
506506
}
507507

508508
enforcer := mountMonitoringSecurityPolicyEnforcer()
509-
err := Mount(context.Background(), 0, 0, target, true, false, nil, nil, enforcer)
509+
err := mount(context.Background(), 0, 0, target, true, false, nil, nil, enforcer)
510510
if err != nil {
511511
t.Fatalf("expected nil err, got: %v", err)
512512
}
@@ -549,7 +549,7 @@ func Test_Read_Write_Security_Policy_Enforcement_Mount_Calls(t *testing.T) {
549549
}
550550

551551
enforcer := mountMonitoringSecurityPolicyEnforcer()
552-
err := Mount(context.Background(), 0, 0, target, false, false, nil, nil, enforcer)
552+
err := mount(context.Background(), 0, 0, target, false, false, nil, nil, enforcer)
553553
if err != nil {
554554
t.Fatalf("expected nil err, got: %v", err)
555555
}
@@ -592,12 +592,12 @@ func Test_Security_Policy_Enforcement_Unmount_Calls(t *testing.T) {
592592
}
593593

594594
enforcer := mountMonitoringSecurityPolicyEnforcer()
595-
err := Mount(context.Background(), 0, 0, target, true, false, nil, nil, enforcer)
595+
err := mount(context.Background(), 0, 0, target, true, false, nil, nil, enforcer)
596596
if err != nil {
597597
t.Fatalf("expected nil err, got: %v", err)
598598
}
599599

600-
err = Unmount(context.Background(), 0, 0, target, false, nil, enforcer)
600+
err = unmount(context.Background(), 0, 0, target, false, nil, enforcer)
601601
if err != nil {
602602
t.Fatalf("expected nil err, got: %v", err)
603603
}
@@ -668,7 +668,7 @@ func Test_CreateVerityTarget_And_Mount_Called_With_Correct_Parameters(t *testing
668668
return nil
669669
}
670670

671-
if err := Mount(
671+
if err := mount(
672672
context.Background(),
673673
0,
674674
0,
@@ -717,7 +717,7 @@ func Test_osMkdirAllFails_And_RemoveDevice_Called(t *testing.T) {
717717
return nil
718718
}
719719

720-
if err := Mount(
720+
if err := mount(
721721
context.Background(),
722722
0,
723723
0,

‎internal/protocol/guestrequest/types.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,16 @@ type RS4NetworkModifyRequest struct {
4141
RequestType RequestType `json:"RequestType,omitempty"`
4242
Settings interface{} `json:"Settings,omitempty"`
4343
}
44+
45+
var (
46+
// V5 GUIDs for SCSI controllers
47+
// These GUIDs are created with namespace GUID "d422512d-2bf2-4752-809d-7b82b5fcb1b4"
48+
// and index as names. For example, first GUID is created like this:
49+
// guid.NewV5("d422512d-2bf2-4752-809d-7b82b5fcb1b4", []byte("0"))
50+
ScsiControllerGuids = []string{
51+
"df6d0690-79e5-55b6-a5ec-c1e2f77f580a",
52+
"0110f83b-de10-5172-a266-78bca56bf50a",
53+
"b5d2d8d4-3a75-51bf-945b-3444dc6b8579",
54+
"305891a9-b251-5dfe-91a2-c25d9212275b",
55+
}
56+
)

0 commit comments

Comments
 (0)