Docker json-file logging driver hangs when rotating files on Windows. Once a container gets in hung state, no more log entries are saved for it unless docker or the container is restarted.
PS> Get-EventLog -LogName Application -Source Docker shows the following error messages:
The issue does not happen every time but happens quite frequently.
> docker inspect e9b
[
{
"Id": "e9b7722458801aadcfdc6f37a7d44c077c36d3680a906a305dad878e9cf6a9ae",
"Created": "REDACTED",
"Path": "REDACTED",
"Args": [
"REDACTED"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 3648,
"ExitCode": 0,
"Error": "",
"StartedAt": "2019-05-28T00:52:02.4274538Z",
"FinishedAt": "2019-05-28T00:52:00.7740957Z"
},
"Image": "sha256:b301392b301239e154b21e03c8b77c03abd14e8ab3459835036a411998b60fd8",
"ResolvConfPath": "",
"HostnamePath": "",
"HostsPath": "",
"LogPath": "C:\\ProgramData\\docker\\containers\\e9b7722458801aadcfdc6f37a7d44c077c36d3680a906a305dad878e9cf6a9ae\\e9b7722458801aadcfdc6f37a7d44c077c36d3680a906a305dad878e9cf6a9ae-json.log",
"Name": "REDACTED",
"RestartCount": 0,
"Driver": "windowsfilter",
"Platform": "windows",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"REDACTED"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "2",
"max-size": "1m"
}
},
"NetworkMode": "nat",
"PortBindings": null,
"RestartPolicy": {
"Name": "unless-stopped",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 0,
"ConsoleSize": [
0,
0
],
"Isolation": "process",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"dir": "C:\\ProgramData\\docker\\windowsfilter\\e9b7722458801aadcfdc6f37a7d44c077c36d3680a906a305dad878e9cf6a9ae"
},
"Name": "windowsfilter"
},
"Mounts": [
{
"Type": "bind",
"Source": "REDACTED",
"Destination": "REDACTED",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "REDACTED",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": null,
"Cmd": [
"REDACTED",
],
"Image": "REDACTED",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": [],
"OnBuild": null,
"Labels": {
"agent.config": "REDACTED",
"agent.managed": "true"
},
"StopSignal": "SIGTERM"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "e9b7722458801aadcfdc6f37a7d44c077c36d3680a906a305dad878e9cf6a9ae",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "e9b7722458801aadcfdc6f37a7d44c077c36d3680a906a305dad878e9cf6a9ae",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"nat": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "93a8f0003991defdec723729ef537a49ecab734ede99e070a8dd1181ff79137e",
"EndpointID": "8d8509776404f0d0532d5afe3132c5a4a63e94e0b5fcb94751d53f24e29e1f9a",
"Gateway": "REDACTED",
"IPAddress": "REDACTED",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "00:15:5d:02:1c:ff",
"DriverOpts": null
}
}
}
}
]
Description
Docker json-file logging driver hangs when rotating files on Windows. Once a container gets in hung state, no more log entries are saved for it unless docker or the container is restarted.
PS> Get-EventLog -LogName Application -Source Docker shows the following error messages:
(REPEATED UNTIL MITIGATED)
The issue does not happen every time but happens quite frequently.
Steps to reproduce the issue:
0. Run a Windows 2019 OS VM with docker using named pipes.
Describe the results you received:
Describe the results you expected:
Additional information you deem important (e.g. issue happens only occasionally):
This issue does not occur every time. But it happens very frequently when multiple containers are running on multiple VMs.
Output of
docker version:Output of
docker info:Additional environment details (AWS, VirtualBox, physical, etc.):
Windows 2019 VM on GCP
Mitigations = restart affected container, or restart docker service on Windows.
docker inspect output on a container which got into this state