Skip to content

Commit 5624dff

Browse files
committed
fix: verify swarm task convergence before marking db deploys done
1 parent d7ea9cf commit 5624dff

7 files changed

Lines changed: 82 additions & 6 deletions

File tree

‎packages/server/src/services/libsql.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ import {
77
} from "@dokploy/server/db/schema";
88
import { generatePassword } from "@dokploy/server/templates";
99
import { buildLibsql } from "@dokploy/server/utils/databases/libsql";
10-
import { pullImage } from "@dokploy/server/utils/docker/utils";
10+
import {
11+
pullImage,
12+
waitForSwarmServiceConvergence,
13+
} from "@dokploy/server/utils/docker/utils";
1114
import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
1215
import { TRPCError } from "@trpc/server";
1316
import { eq, getTableColumns } from "drizzle-orm";
@@ -149,6 +152,7 @@ export const deployLibsql = async (
149152
}
150153

151154
await buildLibsql(libsql);
155+
await waitForSwarmServiceConvergence(libsql.appName, libsql.serverId);
152156
await updateLibsqlById(libsqlId, {
153157
applicationStatus: "done",
154158
});

‎packages/server/src/services/mariadb.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ import {
77
} from "@dokploy/server/db/schema";
88
import { generatePassword } from "@dokploy/server/templates";
99
import { buildMariadb } from "@dokploy/server/utils/databases/mariadb";
10-
import { pullImage } from "@dokploy/server/utils/docker/utils";
10+
import {
11+
pullImage,
12+
waitForSwarmServiceConvergence,
13+
} from "@dokploy/server/utils/docker/utils";
1114
import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
1215
import { TRPCError } from "@trpc/server";
1316
import { eq, getTableColumns } from "drizzle-orm";
@@ -154,6 +157,7 @@ export const deployMariadb = async (
154157
}
155158

156159
await buildMariadb(mariadb);
160+
await waitForSwarmServiceConvergence(mariadb.appName, mariadb.serverId);
157161
await updateMariadbById(mariadbId, {
158162
applicationStatus: "done",
159163
});

‎packages/server/src/services/mongo.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@ import {
88
} from "@dokploy/server/db/schema";
99
import { generatePassword } from "@dokploy/server/templates";
1010
import { buildMongo } from "@dokploy/server/utils/databases/mongo";
11-
import { pullImage } from "@dokploy/server/utils/docker/utils";
11+
import {
12+
pullImage,
13+
waitForSwarmServiceConvergence,
14+
} from "@dokploy/server/utils/docker/utils";
1215
import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
1316
import { TRPCError } from "@trpc/server";
1417
import { eq, getTableColumns } from "drizzle-orm";
@@ -169,6 +172,7 @@ export const deployMongo = async (
169172
}
170173

171174
await buildMongo(mongo);
175+
await waitForSwarmServiceConvergence(mongo.appName, mongo.serverId);
172176
await updateMongoById(mongoId, {
173177
applicationStatus: "done",
174178
});

‎packages/server/src/services/mysql.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ import {
77
} from "@dokploy/server/db/schema";
88
import { generatePassword } from "@dokploy/server/templates";
99
import { buildMysql } from "@dokploy/server/utils/databases/mysql";
10-
import { pullImage } from "@dokploy/server/utils/docker/utils";
10+
import {
11+
pullImage,
12+
waitForSwarmServiceConvergence,
13+
} from "@dokploy/server/utils/docker/utils";
1114
import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
1215
import { TRPCError } from "@trpc/server";
1316
import { eq, getTableColumns } from "drizzle-orm";
@@ -152,6 +155,7 @@ export const deployMySql = async (
152155
}
153156

154157
await buildMysql(mysql);
158+
await waitForSwarmServiceConvergence(mysql.appName, mysql.serverId);
155159
await updateMySqlById(mysqlId, {
156160
applicationStatus: "done",
157161
});

‎packages/server/src/services/postgres.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ import {
77
} from "@dokploy/server/db/schema";
88
import { generatePassword } from "@dokploy/server/templates";
99
import { buildPostgres } from "@dokploy/server/utils/databases/postgres";
10-
import { pullImage } from "@dokploy/server/utils/docker/utils";
10+
import {
11+
pullImage,
12+
waitForSwarmServiceConvergence,
13+
} from "@dokploy/server/utils/docker/utils";
1114
import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
1215
import { TRPCError } from "@trpc/server";
1316
import { eq, getTableColumns } from "drizzle-orm";
@@ -165,6 +168,8 @@ export const deployPostgres = async (
165168

166169
await buildPostgres(postgres);
167170

171+
await waitForSwarmServiceConvergence(postgres.appName, postgres.serverId);
172+
168173
await updatePostgresById(postgresId, {
169174
applicationStatus: "done",
170175
});

‎packages/server/src/services/redis.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ import {
66
} from "@dokploy/server/db/schema";
77
import { generatePassword } from "@dokploy/server/templates";
88
import { buildRedis } from "@dokploy/server/utils/databases/redis";
9-
import { pullImage } from "@dokploy/server/utils/docker/utils";
9+
import {
10+
pullImage,
11+
waitForSwarmServiceConvergence,
12+
} from "@dokploy/server/utils/docker/utils";
1013
import { execAsyncRemote } from "@dokploy/server/utils/process/execAsync";
1114
import { TRPCError } from "@trpc/server";
1215
import { eq } from "drizzle-orm";
@@ -119,6 +122,7 @@ export const deployRedis = async (
119122
}
120123

121124
await buildRedis(redis);
125+
await waitForSwarmServiceConvergence(redis.appName, redis.serverId);
122126
await updateRedisById(redisId, {
123127
applicationStatus: "done",
124128
});

‎packages/server/src/utils/docker/utils.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -934,6 +934,57 @@ const getSwarmServiceContainerId = async (
934934
}
935935
};
936936

937+
export class ServiceConvergenceError extends Error {}
938+
939+
export const waitForSwarmServiceConvergence = async (
940+
appName: string,
941+
serverId?: string | null,
942+
options?: { timeoutMs?: number; intervalMs?: number },
943+
): Promise<void> => {
944+
const timeoutMs = options?.timeoutMs ?? 45_000;
945+
const intervalMs = options?.intervalMs ?? 2_000;
946+
const remoteDocker = await getRemoteDocker(serverId);
947+
const service = remoteDocker.getService(appName);
948+
const deadline = Date.now() + timeoutMs;
949+
950+
let lastState = "unknown";
951+
while (true) {
952+
const info = await service.inspect();
953+
const desiredTasksCount = info.Spec?.Mode?.Replicated?.Replicas ?? 1;
954+
955+
const tasks = await remoteDocker.listTasks({
956+
filters: JSON.stringify({ service: [appName] }),
957+
});
958+
const currentTasks = tasks.filter(
959+
(task) => task.DesiredState === "running",
960+
);
961+
const runningTasksCount = currentTasks.filter(
962+
(task) => task.Status?.State === "running",
963+
).length;
964+
965+
if (runningTasksCount >= desiredTasksCount) {
966+
return;
967+
}
968+
969+
const failedTask = currentTasks.find((task) =>
970+
["failed", "rejected"].includes(task.Status?.State ?? ""),
971+
);
972+
lastState =
973+
failedTask?.Status?.Err ??
974+
failedTask?.Status?.State ??
975+
currentTasks[0]?.Status?.State ??
976+
lastState;
977+
978+
if (Date.now() >= deadline) {
979+
throw new ServiceConvergenceError(
980+
`Service ${appName} did not converge within ${timeoutMs}ms: ${runningTasksCount}/${desiredTasksCount} tasks running (last state: ${lastState})`,
981+
);
982+
}
983+
984+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
985+
}
986+
};
987+
937988
export const checkPostgresHealth = async (): Promise<ServiceHealthStatus> => {
938989
const serviceCheck = await checkSwarmServiceRunning("dokploy-postgres");
939990
if (serviceCheck.status === "unhealthy") {

0 commit comments

Comments
 (0)