Skip to content
This repository was archived by the owner on Jan 20, 2021. It is now read-only.

Commit f69f09f

Browse files
author
Yanqiang Miao
committed
add compress option for 'jsonfiles' log driver
This PR adds support for compressibility of log file. I added a new option conpression for the jsonfile log driver, this option allows the user to specify compression algorithm to compress the log files. By default, the log files will be not compressed. At present, only support 'gzip'. Signed-off-by: Yanqiang Miao <[email protected]> 'docker logs' can read from compressed files Signed-off-by: Yanqiang Miao <[email protected]> Add Metadata to the gzip header, optmize 'readlog' Signed-off-by: Yanqiang Miao <[email protected]>
1 parent 241c904 commit f69f09f

3 files changed

Lines changed: 322 additions & 49 deletions

File tree

‎daemon/logger/jsonfilelog/jsonfilelog.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ func New(info logger.Info) (logger.Logger, error) {
4949
if err != nil {
5050
return nil, err
5151
}
52+
if capval <= 0 {
53+
return nil, fmt.Errorf("max-size should be a positive numbler")
54+
}
5255
}
5356
var maxFiles = 1
5457
if maxFileString, ok := info.Config["max-file"]; ok {
@@ -62,6 +65,18 @@ func New(info logger.Info) (logger.Logger, error) {
6265
}
6366
}
6467

68+
var compress bool
69+
if compressString, ok := info.Config["compress"]; ok {
70+
var err error
71+
compress, err = strconv.ParseBool(compressString)
72+
if err != nil {
73+
return nil, err
74+
}
75+
if compress && (maxFiles == 1 || capval == -1) {
76+
return nil, fmt.Errorf("compress cannot be true when max-file is less than 2 or max-size is not set")
77+
}
78+
}
79+
6580
attrs, err := info.ExtraAttributes(nil)
6681
if err != nil {
6782
return nil, err
@@ -95,7 +110,7 @@ func New(info logger.Info) (logger.Logger, error) {
95110
return b, nil
96111
}
97112

98-
writer, err := loggerutils.NewLogFile(info.LogPath, capval, maxFiles, marshalFunc, decodeFunc, 0640)
113+
writer, err := loggerutils.NewLogFile(info.LogPath, capval, maxFiles, compress, marshalFunc, decodeFunc, 0640)
99114
if err != nil {
100115
return nil, err
101116
}
@@ -139,6 +154,7 @@ func ValidateLogOpt(cfg map[string]string) error {
139154
switch key {
140155
case "max-file":
141156
case "max-size":
157+
case "compress":
142158
case "labels":
143159
case "env":
144160
case "env-regex":

‎daemon/logger/jsonfilelog/jsonfilelog_test.go‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package jsonfilelog // import "github.com/docker/docker/daemon/logger/jsonfilelo
22

33
import (
44
"bytes"
5+
"compress/gzip"
56
"encoding/json"
67
"io/ioutil"
78
"os"
@@ -142,7 +143,7 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
142143
}
143144
defer os.RemoveAll(tmp)
144145
filename := filepath.Join(tmp, "container.log")
145-
config := map[string]string{"max-file": "2", "max-size": "1k"}
146+
config := map[string]string{"max-file": "3", "max-size": "1k", "compress": "true"}
146147
l, err := New(logger.Info{
147148
ContainerID: cid,
148149
LogPath: filename,
@@ -152,21 +153,55 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
152153
t.Fatal(err)
153154
}
154155
defer l.Close()
155-
for i := 0; i < 20; i++ {
156+
for i := 0; i < 36; i++ {
156157
if err := l.Log(&logger.Message{Line: []byte("line" + strconv.Itoa(i)), Source: "src1"}); err != nil {
157158
t.Fatal(err)
158159
}
159160
}
161+
160162
res, err := ioutil.ReadFile(filename)
161163
if err != nil {
162164
t.Fatal(err)
163165
}
166+
164167
penUlt, err := ioutil.ReadFile(filename + ".1")
168+
if err != nil {
169+
if !os.IsNotExist(err) {
170+
t.Fatal(err)
171+
}
172+
173+
file, err := os.Open(filename + ".1.gz")
174+
defer file.Close()
175+
if err != nil {
176+
t.Fatal(err)
177+
}
178+
zipReader, err := gzip.NewReader(file)
179+
defer zipReader.Close()
180+
if err != nil {
181+
t.Fatal(err)
182+
}
183+
penUlt, err = ioutil.ReadAll(zipReader)
184+
if err != nil {
185+
t.Fatal(err)
186+
}
187+
}
188+
189+
file, err := os.Open(filename + ".2.gz")
190+
defer file.Close()
191+
if err != nil {
192+
t.Fatal(err)
193+
}
194+
zipReader, err := gzip.NewReader(file)
195+
defer zipReader.Close()
196+
if err != nil {
197+
t.Fatal(err)
198+
}
199+
antepenult, err := ioutil.ReadAll(zipReader)
165200
if err != nil {
166201
t.Fatal(err)
167202
}
168203

169-
expectedPenultimate := `{"log":"line0\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
204+
expectedAntepenultimate := `{"log":"line0\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
170205
{"log":"line1\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
171206
{"log":"line2\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
172207
{"log":"line3\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
@@ -183,10 +218,27 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
183218
{"log":"line14\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
184219
{"log":"line15\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
185220
`
186-
expected := `{"log":"line16\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
221+
expectedPenultimate := `{"log":"line16\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
187222
{"log":"line17\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
188223
{"log":"line18\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
189224
{"log":"line19\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
225+
{"log":"line20\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
226+
{"log":"line21\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
227+
{"log":"line22\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
228+
{"log":"line23\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
229+
{"log":"line24\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
230+
{"log":"line25\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
231+
{"log":"line26\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
232+
{"log":"line27\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
233+
{"log":"line28\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
234+
{"log":"line29\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
235+
{"log":"line30\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
236+
{"log":"line31\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
237+
`
238+
expected := `{"log":"line32\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
239+
{"log":"line33\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
240+
{"log":"line34\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
241+
{"log":"line35\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
190242
`
191243

192244
if string(res) != expected {
@@ -195,7 +247,9 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
195247
if string(penUlt) != expectedPenultimate {
196248
t.Fatalf("Wrong log content: %q, expected %q", penUlt, expectedPenultimate)
197249
}
198-
250+
if string(antepenult) != expectedAntepenultimate {
251+
t.Fatalf("Wrong log content: %q, expected %q", antepenult, expectedAntepenultimate)
252+
}
199253
}
200254

201255
func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {

0 commit comments

Comments
 (0)