Skip to content

Commit 831df67

Browse files
dpernydperny
authored
Merge pull request #2571 from abhi/master
Adding logic to restore networks in order
2 parents 5b8996c + 7f9db67 commit 831df67

2 files changed

Lines changed: 308 additions & 41 deletions

File tree

‎manager/allocator/allocator_test.go‎

Lines changed: 242 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -563,16 +563,35 @@ func TestNoDuplicateIPs(t *testing.T) {
563563
},
564564
Ingress: true,
565565
},
566+
IPAM: &api.IPAMOptions{
567+
Driver: &api.Driver{},
568+
Configs: []*api.IPAMConfig{
569+
{
570+
Subnet: "10.0.0.0/24",
571+
Gateway: "10.0.0.1",
572+
},
573+
},
574+
},
575+
DriverState: &api.Driver{},
566576
}
567577
assert.NoError(t, store.CreateNetwork(tx, in))
568-
569578
n1 := &api.Network{
570579
ID: "testID1",
571580
Spec: api.NetworkSpec{
572581
Annotations: api.Annotations{
573582
Name: "test1",
574583
},
575584
},
585+
IPAM: &api.IPAMOptions{
586+
Driver: &api.Driver{},
587+
Configs: []*api.IPAMConfig{
588+
{
589+
Subnet: "10.1.0.0/24",
590+
Gateway: "10.1.0.1",
591+
},
592+
},
593+
},
594+
DriverState: &api.Driver{},
576595
}
577596
assert.NoError(t, store.CreateNetwork(tx, n1))
578597

@@ -649,7 +668,6 @@ func TestNoDuplicateIPs(t *testing.T) {
649668

650669
return nil
651670
}))
652-
653671
a, err := New(s, nil)
654672
assert.NoError(t, err)
655673
assert.NotNil(t, a)
@@ -661,7 +679,6 @@ func TestNoDuplicateIPs(t *testing.T) {
661679

662680
// Confirm task gets a unique IP
663681
watchTask(t, s, taskWatch, false, hasUniqueIP)
664-
665682
a.Stop()
666683
}
667684
}
@@ -682,6 +699,15 @@ func TestAllocatorRestoreForDuplicateIPs(t *testing.T) {
682699
},
683700
Ingress: true,
684701
},
702+
IPAM: &api.IPAMOptions{
703+
Driver: &api.Driver{},
704+
Configs: []*api.IPAMConfig{
705+
{
706+
Subnet: "10.0.0.0/24",
707+
Gateway: "10.0.0.1",
708+
},
709+
},
710+
},
685711
}
686712
assert.NoError(t, store.CreateNetwork(tx, in))
687713

@@ -815,6 +841,16 @@ func TestAllocatorRestartNoEndpointSpec(t *testing.T) {
815841
Name: "net1",
816842
},
817843
},
844+
IPAM: &api.IPAMOptions{
845+
Driver: &api.Driver{},
846+
Configs: []*api.IPAMConfig{
847+
{
848+
Subnet: "10.0.0.0/24",
849+
Gateway: "10.0.0.1",
850+
},
851+
},
852+
},
853+
DriverState: &api.Driver{},
818854
}
819855
assert.NoError(t, store.CreateNetwork(tx, in))
820856

@@ -887,7 +923,6 @@ func TestAllocatorRestartNoEndpointSpec(t *testing.T) {
887923
hasNoIPOverlapServices := func(fakeT assert.TestingT, service *api.Service) bool {
888924
assert.NotEqual(fakeT, len(service.Endpoint.VirtualIPs), 0)
889925
assert.NotEqual(fakeT, len(service.Endpoint.VirtualIPs[0].Addr), 0)
890-
891926
assignedVIP := service.Endpoint.VirtualIPs[0].Addr
892927
if assignedIPs[assignedVIP] {
893928
t.Fatalf("service %s assigned duplicate IP %s", service.ID, assignedVIP)
@@ -903,7 +938,6 @@ func TestAllocatorRestartNoEndpointSpec(t *testing.T) {
903938
hasNoIPOverlapTasks := func(fakeT assert.TestingT, s *store.MemoryStore, task *api.Task) bool {
904939
assert.NotEqual(fakeT, len(task.Networks), 0)
905940
assert.NotEqual(fakeT, len(task.Networks[0].Addresses), 0)
906-
907941
assignedIP := task.Networks[0].Addresses[0]
908942
if assignedIPs[assignedIP] {
909943
t.Fatalf("task %s assigned duplicate IP %s", task.ID, assignedIP)
@@ -939,6 +973,209 @@ func TestAllocatorRestartNoEndpointSpec(t *testing.T) {
939973
assert.Len(t, expectedIPs, 0)
940974
}
941975

976+
// TestAllocatorRestoreForUnallocatedNetwork tests allocator restart
977+
// scenarios where there is a combination of allocated and unallocated
978+
// networks and tests whether the restore logic ensures the networks
979+
// services and tasks that were preallocated are allocated correctly
980+
// followed by the allocation of unallocated networks prior to the
981+
// restart.
982+
func TestAllocatorRestoreForUnallocatedNetwork(t *testing.T) {
983+
s := store.NewMemoryStore(nil)
984+
assert.NotNil(t, s)
985+
defer s.Close()
986+
// Create 3 services with 1 task each
987+
numsvcstsks := 3
988+
var n1 *api.Network
989+
var n2 *api.Network
990+
assert.NoError(t, s.Update(func(tx store.Tx) error {
991+
// populate ingress network
992+
in := &api.Network{
993+
ID: "ingress-nw-id",
994+
Spec: api.NetworkSpec{
995+
Annotations: api.Annotations{
996+
Name: "default-ingress",
997+
},
998+
Ingress: true,
999+
},
1000+
IPAM: &api.IPAMOptions{
1001+
Driver: &api.Driver{},
1002+
Configs: []*api.IPAMConfig{
1003+
{
1004+
Subnet: "10.0.0.0/24",
1005+
Gateway: "10.0.0.1",
1006+
},
1007+
},
1008+
},
1009+
}
1010+
assert.NoError(t, store.CreateNetwork(tx, in))
1011+
1012+
n1 = &api.Network{
1013+
ID: "testID1",
1014+
Spec: api.NetworkSpec{
1015+
Annotations: api.Annotations{
1016+
Name: "test1",
1017+
},
1018+
},
1019+
IPAM: &api.IPAMOptions{
1020+
Driver: &api.Driver{},
1021+
Configs: []*api.IPAMConfig{
1022+
{
1023+
Subnet: "10.1.0.0/24",
1024+
Gateway: "10.1.0.1",
1025+
},
1026+
},
1027+
},
1028+
DriverState: &api.Driver{},
1029+
}
1030+
assert.NoError(t, store.CreateNetwork(tx, n1))
1031+
1032+
n2 = &api.Network{
1033+
// Intentionally named testID0 so that in restore this network
1034+
// is looked into first
1035+
ID: "testID0",
1036+
Spec: api.NetworkSpec{
1037+
Annotations: api.Annotations{
1038+
Name: "test2",
1039+
},
1040+
},
1041+
}
1042+
assert.NoError(t, store.CreateNetwork(tx, n2))
1043+
1044+
for i := 0; i != numsvcstsks; i++ {
1045+
svc := &api.Service{
1046+
ID: "testServiceID" + strconv.Itoa(i),
1047+
Spec: api.ServiceSpec{
1048+
Annotations: api.Annotations{
1049+
Name: "service" + strconv.Itoa(i),
1050+
},
1051+
Task: api.TaskSpec{
1052+
Networks: []*api.NetworkAttachmentConfig{
1053+
{
1054+
Target: "testID1",
1055+
},
1056+
},
1057+
},
1058+
Endpoint: &api.EndpointSpec{
1059+
Mode: api.ResolutionModeVirtualIP,
1060+
Ports: []*api.PortConfig{
1061+
{
1062+
Name: "",
1063+
Protocol: api.ProtocolTCP,
1064+
TargetPort: 8000,
1065+
PublishedPort: uint32(8001 + i),
1066+
},
1067+
},
1068+
},
1069+
},
1070+
Endpoint: &api.Endpoint{
1071+
Ports: []*api.PortConfig{
1072+
{
1073+
Name: "",
1074+
Protocol: api.ProtocolTCP,
1075+
TargetPort: 8000,
1076+
PublishedPort: uint32(8001 + i),
1077+
},
1078+
},
1079+
VirtualIPs: []*api.Endpoint_VirtualIP{
1080+
{
1081+
NetworkID: "ingress-nw-id",
1082+
Addr: "10.0.0." + strconv.Itoa(2+i) + "/24",
1083+
},
1084+
{
1085+
NetworkID: "testID1",
1086+
Addr: "10.1.0." + strconv.Itoa(2+i) + "/24",
1087+
},
1088+
},
1089+
},
1090+
}
1091+
assert.NoError(t, store.CreateService(tx, svc))
1092+
}
1093+
return nil
1094+
}))
1095+
1096+
for i := 0; i != numsvcstsks; i++ {
1097+
assert.NoError(t, s.Update(func(tx store.Tx) error {
1098+
tsk := &api.Task{
1099+
ID: "testTaskID" + strconv.Itoa(i),
1100+
Status: api.TaskStatus{
1101+
State: api.TaskStateNew,
1102+
},
1103+
Spec: api.TaskSpec{
1104+
Networks: []*api.NetworkAttachmentConfig{
1105+
{
1106+
Target: "testID1",
1107+
},
1108+
},
1109+
},
1110+
ServiceID: "testServiceID" + strconv.Itoa(i),
1111+
DesiredState: api.TaskStateRunning,
1112+
}
1113+
assert.NoError(t, store.CreateTask(tx, tsk))
1114+
return nil
1115+
}))
1116+
}
1117+
1118+
assignedIPs := make(map[string]bool)
1119+
expectedIPs := map[string]string{
1120+
"testServiceID0": "10.1.0.2/24",
1121+
"testServiceID1": "10.1.0.3/24",
1122+
"testServiceID2": "10.1.0.4/24",
1123+
"testTaskID0": "10.1.0.5/24",
1124+
"testTaskID1": "10.1.0.6/24",
1125+
"testTaskID2": "10.1.0.7/24",
1126+
}
1127+
hasNoIPOverlapServices := func(fakeT assert.TestingT, service *api.Service) bool {
1128+
assert.NotEqual(fakeT, len(service.Endpoint.VirtualIPs), 0)
1129+
assert.NotEqual(fakeT, len(service.Endpoint.VirtualIPs[0].Addr), 0)
1130+
assignedVIP := service.Endpoint.VirtualIPs[1].Addr
1131+
if assignedIPs[assignedVIP] {
1132+
t.Fatalf("service %s assigned duplicate IP %s", service.ID, assignedVIP)
1133+
}
1134+
assignedIPs[assignedVIP] = true
1135+
ip, ok := expectedIPs[service.ID]
1136+
assert.True(t, ok)
1137+
assert.Equal(t, ip, assignedVIP)
1138+
delete(expectedIPs, service.ID)
1139+
return true
1140+
}
1141+
1142+
hasNoIPOverlapTasks := func(fakeT assert.TestingT, s *store.MemoryStore, task *api.Task) bool {
1143+
assert.NotEqual(fakeT, len(task.Networks), 0)
1144+
assert.NotEqual(fakeT, len(task.Networks[0].Addresses), 0)
1145+
assignedIP := task.Networks[1].Addresses[0]
1146+
if assignedIPs[assignedIP] {
1147+
t.Fatalf("task %s assigned duplicate IP %s", task.ID, assignedIP)
1148+
}
1149+
assignedIPs[assignedIP] = true
1150+
ip, ok := expectedIPs[task.ID]
1151+
assert.True(t, ok)
1152+
assert.Equal(t, ip, assignedIP)
1153+
delete(expectedIPs, task.ID)
1154+
return true
1155+
}
1156+
1157+
a, err := New(s, nil)
1158+
assert.NoError(t, err)
1159+
assert.NotNil(t, a)
1160+
// Start allocator
1161+
go func() {
1162+
assert.NoError(t, a.Run(context.Background()))
1163+
}()
1164+
defer a.Stop()
1165+
1166+
taskWatch, cancel := state.Watch(s.WatchQueue(), api.EventUpdateTask{}, api.EventDeleteTask{})
1167+
defer cancel()
1168+
1169+
serviceWatch, cancel := state.Watch(s.WatchQueue(), api.EventUpdateService{}, api.EventDeleteService{})
1170+
defer cancel()
1171+
1172+
// Confirm tasks have no IPs that overlap with the services VIPs on restart
1173+
for i := 0; i != numsvcstsks; i++ {
1174+
watchTask(t, s, taskWatch, false, hasNoIPOverlapTasks)
1175+
watchService(t, serviceWatch, false, hasNoIPOverlapServices)
1176+
}
1177+
}
1178+
9421179
func TestNodeAllocator(t *testing.T) {
9431180
s := store.NewMemoryStore(nil)
9441181
assert.NotNil(t, s)

0 commit comments

Comments
 (0)