Skip to content

Commit 046824c

Browse files
waybackarchiverwaybackarchiver
authoredOct 27, 2022
Set the default path for the reduxer (#235)
* Set the default path for the reduxer * Separate TestWayback from `service/utils_test.go`
1 parent c67b523 commit 046824c

File tree

5 files changed

+121
-54
lines changed

5 files changed

+121
-54
lines changed
 

‎cmd/wayback/wayback.go

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ func assets(art reduxer.Artifact) []reduxer.Asset {
3030
}
3131

3232
func archive(cmd *cobra.Command, args []string) {
33+
// TODO: clean the auto-created temporary directory.
3334
archiving := func(ctx context.Context, urls []*url.URL) error {
3435
g, ctx := errgroup.WithContext(ctx)
3536
rdx, err := reduxer.Do(ctx, urls...)

‎config/config_test.go

+28-12
Original file line numberDiff line numberDiff line change
@@ -1370,20 +1370,36 @@ func TestBoltPath(t *testing.T) {
13701370
}
13711371

13721372
func TestStorageDir(t *testing.T) {
1373-
dir := os.TempDir()
1373+
var tests = []struct {
1374+
dir string
1375+
exp string
1376+
}{
1377+
{
1378+
dir: "",
1379+
exp: defStorageDir,
1380+
},
1381+
{
1382+
dir: "/path/to/storage",
1383+
exp: "/path/to/storage",
1384+
},
1385+
}
13741386

1375-
os.Clearenv()
1376-
os.Setenv("WAYBACK_STORAGE_DIR", dir)
1387+
for _, test := range tests {
1388+
t.Run("", func(t *testing.T) {
1389+
os.Clearenv()
1390+
os.Setenv("WAYBACK_STORAGE_DIR", test.dir)
13771391

1378-
parser := NewParser()
1379-
opts, err := parser.ParseEnvironmentVariables()
1380-
if err != nil {
1381-
t.Fatalf(`Parsing environment variables failed: %v`, err)
1382-
}
1392+
parser := NewParser()
1393+
opts, err := parser.ParseEnvironmentVariables()
1394+
if err != nil {
1395+
t.Fatalf(`Parsing environment variables failed: %v`, err)
1396+
}
13831397

1384-
got := opts.StorageDir()
1385-
if got != dir {
1386-
t.Fatalf(`Unexpected storage binary directory got %s instead of %s`, got, dir)
1398+
got := opts.StorageDir()
1399+
if got != test.exp {
1400+
t.Errorf(`Unexpected storage binary directory got %s instead of %s`, got, test.dir)
1401+
}
1402+
})
13871403
}
13881404
}
13891405

@@ -1394,7 +1410,7 @@ func TestEnabledReduxer(t *testing.T) {
13941410
}{
13951411
{
13961412
dir: "",
1397-
exp: false,
1413+
exp: true,
13981414
},
13991415
{
14001416
dir: "/path/to/storage",

‎config/options.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ package config // import "github.com/wabarc/wayback/config"
66

77
import (
88
"net/url"
9+
"os"
10+
"path"
911
"strings"
1012
"time"
1113

@@ -76,7 +78,6 @@ const (
7678
defEnabledChromeRemote = false
7779
defBoltPathname = "wayback.db"
7880
defPoolingSize = 3
79-
defStorageDir = ""
8081
defMaxMediaSize = "512MB"
8182
defWaybackTimeout = 300
8283
defWaybackMaxRetries = 2
@@ -96,6 +97,7 @@ var (
9697
IPFSToken = ""
9798
IPFSTarget = "web3storage"
9899

100+
defStorageDir = path.Join(os.TempDir(), "reduxer")
99101
defTorRemotePorts = []int{80}
100102
)
101103

@@ -690,7 +692,7 @@ func (o *Options) StorageDir() string {
690692

691693
// EnabledReduxer returns whether enable store binary file locally.
692694
func (o *Options) EnabledReduxer() bool {
693-
return o.storageDir != ""
695+
return o.StorageDir() != ""
694696
}
695697

696698
// MaxMediaSize returns max size to limit download stream media.

‎service/service_test.go

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Copyright 2022 Wayback Archiver. All rights reserved.
2+
// Use of this source code is governed by the GNU GPL v3
3+
// license that can be found in the LICENSE file.
4+
5+
package service // import "github.com/wabarc/wayback/service"
6+
7+
import (
8+
"context"
9+
"net/url"
10+
"os"
11+
"testing"
12+
"time"
13+
14+
"github.com/wabarc/helper"
15+
"github.com/wabarc/logger"
16+
"github.com/wabarc/wayback"
17+
"github.com/wabarc/wayback/config"
18+
"github.com/wabarc/wayback/reduxer"
19+
)
20+
21+
func TestWayback(t *testing.T) {
22+
defer helper.CheckTest(t)
23+
24+
// Don't wayback to any slot to speed up testing.
25+
os.Clearenv()
26+
os.Setenv("WAYBACK_ENABLE_IA", "false")
27+
os.Setenv("WAYBACK_ENABLE_IS", "false")
28+
os.Setenv("WAYBACK_ENABLE_IP", "false")
29+
os.Setenv("WAYBACK_ENABLE_PH", "false")
30+
31+
parser := config.NewParser()
32+
var err error
33+
if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
34+
t.Fatalf("Parse environment variables or flags failed, error: %v", err)
35+
}
36+
logger.SetLogLevel(logger.LevelFatal)
37+
38+
u, _ := url.Parse("https://example.com/")
39+
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
40+
defer cancel()
41+
defer helper.CheckContext(ctx, t)
42+
43+
urls := []*url.URL{u}
44+
do := func(cols []wayback.Collect, rdx reduxer.Reduxer) error {
45+
time.Sleep(3 * time.Second)
46+
return nil
47+
}
48+
w := Wayback(ctx, urls, do)
49+
50+
if w == nil {
51+
t.Fatal("Unexpected wayback exceeded")
52+
}
53+
}
54+
55+
func TestWaybackWithoutReduxer(t *testing.T) {
56+
defer helper.CheckTest(t)
57+
58+
// Don't wayback to any slot to speed up testing.
59+
os.Clearenv()
60+
os.Setenv("WAYBACK_ENABLE_IA", "false")
61+
os.Setenv("WAYBACK_ENABLE_IS", "false")
62+
os.Setenv("WAYBACK_ENABLE_IP", "false")
63+
os.Setenv("WAYBACK_ENABLE_PH", "false")
64+
os.Setenv("WAYBACK_STORAGE_DIR", "")
65+
66+
parser := config.NewParser()
67+
var err error
68+
if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
69+
t.Fatalf("Parse environment variables or flags failed, error: %v", err)
70+
}
71+
logger.SetLogLevel(logger.LevelFatal)
72+
73+
u, _ := url.Parse("https://example.com/")
74+
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
75+
defer cancel()
76+
defer helper.CheckContext(ctx, t)
77+
78+
urls := []*url.URL{u}
79+
do := func(cols []wayback.Collect, rdx reduxer.Reduxer) error {
80+
time.Sleep(3 * time.Second)
81+
return nil
82+
}
83+
w := Wayback(ctx, urls, do)
84+
85+
if w == nil {
86+
t.Fatal("Unexpected wayback exceeded")
87+
}
88+
}

‎service/utils_test.go

-40
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,13 @@
55
package service // import "github.com/wabarc/wayback/service"
66

77
import (
8-
"context"
98
"net/url"
10-
"os"
119
"reflect"
1210
"strconv"
1311
"testing"
14-
"time"
1512

1613
"github.com/wabarc/helper"
17-
"github.com/wabarc/logger"
18-
"github.com/wabarc/wayback"
1914
"github.com/wabarc/wayback/config"
20-
"github.com/wabarc/wayback/reduxer"
2115
)
2216

2317
func TestMatchURL(t *testing.T) {
@@ -114,37 +108,3 @@ func TestExcludeURL(t *testing.T) {
114108
})
115109
}
116110
}
117-
118-
func TestWayback(t *testing.T) {
119-
defer helper.CheckTest(t)
120-
121-
// Don't wayback to any slot to speed up testing.
122-
os.Clearenv()
123-
os.Setenv("WAYBACK_ENABLE_IA", "false")
124-
os.Setenv("WAYBACK_ENABLE_IS", "false")
125-
os.Setenv("WAYBACK_ENABLE_IP", "false")
126-
os.Setenv("WAYBACK_ENABLE_PH", "false")
127-
128-
parser := config.NewParser()
129-
var err error
130-
if config.Opts, err = parser.ParseEnvironmentVariables(); err != nil {
131-
t.Fatalf("Parse environment variables or flags failed, error: %v", err)
132-
}
133-
logger.SetLogLevel(logger.LevelFatal)
134-
135-
u, _ := url.Parse("https://example.com/")
136-
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
137-
defer cancel()
138-
defer helper.CheckContext(ctx, t)
139-
140-
urls := []*url.URL{u}
141-
do := func(cols []wayback.Collect, rdx reduxer.Reduxer) error {
142-
time.Sleep(3 * time.Second)
143-
return nil
144-
}
145-
w := Wayback(ctx, urls, do)
146-
147-
if w == nil {
148-
t.Fatal("Unexpected wayback exceeded")
149-
}
150-
}

0 commit comments

Comments
 (0)