Skip to content

Commit 2598a62

Browse files
committed
fix(volume-backups): restart services after backup failure
1 parent 8bc1bd3 commit 2598a62

2 files changed

Lines changed: 98 additions & 16 deletions

File tree

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import { spawnSync } from "node:child_process";
2+
import { createRestartSafeBackupCommand } from "@dokploy/server/utils/volume-backups/backup";
3+
import { describe, expect, it } from "vitest";
4+
5+
const runCommand = (command: string) =>
6+
spawnSync("bash", ["-c", command], {
7+
encoding: "utf8",
8+
});
9+
10+
const outputLines = (stdout: string) =>
11+
stdout
12+
.trim()
13+
.split("\n")
14+
.filter((line) => line.length > 0);
15+
16+
describe("createRestartSafeBackupCommand", () => {
17+
it("restarts the service and preserves the backup error status", () => {
18+
const result = runCommand(
19+
createRestartSafeBackupCommand({
20+
stopCommand: 'echo "stop"',
21+
backupCommand: 'echo "backup"; exit 23',
22+
startCommand: 'echo "start"',
23+
uploadCommand: 'echo "upload"',
24+
}),
25+
);
26+
27+
expect(result.status).toBe(23);
28+
expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]);
29+
});
30+
31+
it("uploads only after a successful backup and service restart", () => {
32+
const result = runCommand(
33+
createRestartSafeBackupCommand({
34+
stopCommand: 'echo "stop"',
35+
backupCommand: 'echo "backup"',
36+
startCommand: 'echo "start"',
37+
uploadCommand: 'echo "upload"',
38+
}),
39+
);
40+
41+
expect(result.status).toBe(0);
42+
expect(outputLines(result.stdout)).toEqual([
43+
"stop",
44+
"backup",
45+
"start",
46+
"upload",
47+
]);
48+
});
49+
});

‎packages/server/src/utils/volume-backups/backup.ts‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,33 @@ import {
99
normalizeS3Path,
1010
} from "../backups/utils";
1111

12+
interface RestartSafeBackupCommandOptions {
13+
stopCommand: string;
14+
backupCommand: string;
15+
startCommand: string;
16+
uploadCommand: string;
17+
}
18+
19+
export const createRestartSafeBackupCommand = ({
20+
stopCommand,
21+
backupCommand,
22+
startCommand,
23+
uploadCommand,
24+
}: RestartSafeBackupCommandOptions) => `
25+
${stopCommand}
26+
set +e
27+
(
28+
${backupCommand}
29+
)
30+
DOKPLOY_VOLUME_BACKUP_STATUS=$?
31+
set -e
32+
${startCommand}
33+
if [ "$DOKPLOY_VOLUME_BACKUP_STATUS" -ne 0 ]; then
34+
exit "$DOKPLOY_VOLUME_BACKUP_STATUS"
35+
fi
36+
${uploadCommand}
37+
`;
38+
1239
export const getVolumeServiceAppName = (
1340
volumeBackup: Awaited<ReturnType<typeof findVolumeBackupById>>,
1441
): string => {
@@ -117,16 +144,20 @@ export const backupVolume = async (
117144
);
118145

119146
if (serviceType === "application") {
120-
return lockWrapper(`
121-
echo "Stopping application to 0 replicas"
122-
ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}")
123-
echo "Actual replicas: $ACTUAL_REPLICAS"
124-
docker service update --replicas=0 ${volumeBackup.application?.appName}
125-
${backupCommand}
126-
echo "Starting application to $ACTUAL_REPLICAS replicas"
127-
docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}
128-
${uploadCommand}
129-
`);
147+
return lockWrapper(
148+
createRestartSafeBackupCommand({
149+
stopCommand: `
150+
echo "Stopping application to 0 replicas"
151+
ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}")
152+
echo "Actual replicas: $ACTUAL_REPLICAS"
153+
docker service update --replicas=0 ${volumeBackup.application?.appName}`,
154+
backupCommand,
155+
startCommand: `
156+
echo "Starting application to $ACTUAL_REPLICAS replicas"
157+
docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}`,
158+
uploadCommand,
159+
}),
160+
);
130161
}
131162
if (serviceType === "compose") {
132163
const compose = await findComposeById(
@@ -158,11 +189,13 @@ export const backupVolume = async (
158189
echo "Compose container started"
159190
`;
160191
}
161-
return lockWrapper(`
162-
${stopCommand}
163-
${backupCommand}
164-
${startCommand}
165-
${uploadCommand}
166-
`);
192+
return lockWrapper(
193+
createRestartSafeBackupCommand({
194+
stopCommand,
195+
backupCommand,
196+
startCommand,
197+
uploadCommand,
198+
}),
199+
);
167200
}
168201
};

0 commit comments

Comments
 (0)