Skip to content

Commit adc7d45

Browse files
waybackarchiverwaybackarchiver
committed
Starts http service as clear web if missing tor
1 parent 9e3c2e1 commit adc7d45

File tree

8 files changed

+120
-31
lines changed

8 files changed

+120
-31
lines changed

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ You can also specify configuration options either via command flags or via envir
204204
| - | `LOG_TIME` | `true` | Display the date and time in log messages |
205205
| - | `LOG_LEVEL` | `info` | Log level, supported level are `debug`, `info`, `warn`, `error`, `fatal`, defaults to `info` |
206206
| - | `ENABLE_METRICS` | `false` | Enable metrics collector |
207+
| - | `HTTP_LISTEN_ADDR` | `127.0.0.1:8964` | The listen address for the HTTP server |
207208
| - | `CHROME_REMOTE_ADDR` | - | Chrome/Chromium remote debugging address, for screenshot |
208209
| - | `WAYBACK_POOLING_SIZE` | `3` | Number of worker pool for wayback at once |
209210
| - | `WAYBACK_BOLT_PATH` | `./wayback.db` | File path of bolt database |

‎cmd/wayback/serve.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func (srv *service) run(ctx context.Context, store *storage.Storage, pool poolin
9595
go func(errCh chan error) {
9696
errCh <- matrix.Serve()
9797
}(srv.errCh)
98-
case "web":
98+
case "web", "httpd":
9999
tor := httpd.New(ctx, store, pool)
100100
go func(errCh chan error) {
101101
errCh <- tor.Serve()

‎config/config_test.go

+40-4
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ func TestTorLocalPort(t *testing.T) {
396396
got := opts.TorLocalPort()
397397

398398
if got != expected {
399-
t.Fatalf(`Unexpected Tor private key, got %v instead of %q`, got, expected)
399+
t.Fatalf(`Unexpected Tor local port, got %v instead of %q`, got, expected)
400400
}
401401
}
402402

@@ -413,7 +413,7 @@ func TestDefaultTorLocalPortValue(t *testing.T) {
413413
got := opts.TorLocalPort()
414414

415415
if got != expected {
416-
t.Fatalf(`Unexpected Tor private key, got %v instead of %q`, got, expected)
416+
t.Fatalf(`Unexpected Tor local port, got %v instead of %q`, got, expected)
417417
}
418418
}
419419

@@ -431,7 +431,43 @@ func TestTorRemotePorts(t *testing.T) {
431431
got := opts.TorRemotePorts()
432432

433433
if got == nil || len(got) != 3 {
434-
t.Fatalf(`Unexpected Tor private key, got %v instead of %v`, got, expected)
434+
t.Fatalf(`Unexpected Tor remote port, got %v instead of %v`, got, expected)
435+
}
436+
}
437+
438+
func TestListenAddr(t *testing.T) {
439+
t.Parallel()
440+
441+
var tests = []struct {
442+
address string
443+
expected string
444+
}{
445+
{
446+
address: "",
447+
expected: defListenAddr,
448+
},
449+
{
450+
address: defListenAddr,
451+
expected: defListenAddr,
452+
},
453+
}
454+
455+
for _, test := range tests {
456+
t.Run(test.address, func(t *testing.T) {
457+
os.Clearenv()
458+
os.Setenv("HTTP_LISTEN_ADDR", test.address)
459+
460+
parser := NewParser()
461+
opts, err := parser.ParseEnvironmentVariables()
462+
if err != nil {
463+
t.Fatalf(`Parsing failure: %v`, err)
464+
}
465+
466+
result := opts.ListenAddr()
467+
if result != test.expected {
468+
t.Fatalf(`Unexpected LISTEN_ADDR value, got %q instead of %q`, result, test.expected)
469+
}
470+
})
435471
}
436472
}
437473

@@ -448,7 +484,7 @@ func TestDefaultTorRemotePortsValue(t *testing.T) {
448484
got := opts.TorRemotePorts()
449485

450486
if got == nil || len(got) != 1 {
451-
t.Fatalf(`Unexpected Tor private key, got %v instead of %v`, got, expected)
487+
t.Fatalf(`Unexpected Tor remote port, got %v instead of %v`, got, expected)
452488
}
453489
}
454490

‎config/options.go

+8
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const (
6161
defMatrixPassword = ""
6262

6363
defTorPrivateKey = ""
64+
defListenAddr = "127.0.0.1:8964"
6465
defTorLocalPort = 8964
6566
defTorrcFile = "/etc/tor/torrc"
6667

@@ -96,6 +97,7 @@ type Options struct {
9697
irc *irc
9798
tor *tor
9899

100+
listenAddr string
99101
chromeRemoteAddr string
100102
enabledChromeRemote bool
101103
boltPathname string
@@ -180,6 +182,7 @@ func NewOptions() *Options {
180182
logLevel: defLogLevel,
181183
overTor: defOverTor,
182184
metrics: defMetrics,
185+
listenAddr: defListenAddr,
183186
chromeRemoteAddr: defChromeRemoteAddr,
184187
enabledChromeRemote: defEnabledChromeRemote,
185188
boltPathname: defBoltPathname,
@@ -563,6 +566,11 @@ func (o *Options) TorrcFile() string {
563566
return o.tor.torrcFile
564567
}
565568

569+
// ListenAddr returns the listen address for the HTTP server.
570+
func (o *Options) ListenAddr() string {
571+
return o.listenAddr
572+
}
573+
566574
// EnabledChromeRemote returns whether enable Chrome/Chromium remote debugging
567575
// for screenshot
568576
func (o *Options) EnabledChromeRemote() bool {

‎config/parser.go

+2
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
8888
p.opts.logLevel = parseString(val, defLogLevel)
8989
case "ENABLE_METRICS":
9090
p.opts.metrics = parseBool(val, defMetrics)
91+
case "HTTP_LISTEN_ADDR":
92+
p.opts.listenAddr = parseString(val, defListenAddr)
9193
case "CHROME_REMOTE_ADDR":
9294
p.opts.enabledChromeRemote = hasValue(val, defEnabledChromeRemote)
9395
p.opts.chromeRemoteAddr = parseString(val, defChromeRemoteAddr)

‎service/httpd/tor.go

+61-26
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,48 @@ func (t *Tor) Serve() error {
6060
// Start tor with some defaults + elevated verbosity
6161
logger.Info("starting and registering onion service, please wait a bit...")
6262

63-
if _, err := exec.LookPath("tor"); err != nil {
64-
logger.Fatal("%v", err)
63+
handler := newWeb().handle(t.pool)
64+
server := &http.Server{
65+
ReadTimeout: 5 * time.Minute,
66+
WriteTimeout: 5 * time.Minute,
67+
IdleTimeout: 5 * time.Minute,
68+
Handler: handler,
69+
}
70+
71+
switch {
72+
case torExist():
73+
logger.Info("start a tor hidden server")
74+
t.startTorServer(server)
75+
default:
76+
logger.Info("start a clear web server")
77+
server.Addr = config.Opts.ListenAddr()
78+
startHTTPServer(server)
79+
}
80+
81+
<-t.ctx.Done()
82+
logger.Info("stopping tor hidden service...")
83+
if err := server.Shutdown(t.ctx); err != nil {
84+
logger.Error("shutdown tor hidden service failed: %v", err)
85+
return err
86+
}
87+
88+
return errors.New("done")
89+
}
90+
91+
func (t *Tor) torrc() string {
92+
if config.Opts.TorrcFile() == "" {
93+
return ""
94+
}
95+
if torPortBusy() {
96+
return ""
6597
}
98+
if _, err := os.Open(config.Opts.TorrcFile()); err != nil {
99+
return ""
100+
}
101+
return config.Opts.TorrcFile()
102+
}
66103

104+
func (t *Tor) startTorServer(server *http.Server) {
67105
var pvk ed25519.PrivateKey
68106
if config.Opts.TorPrivKey() == "" {
69107
if keypair, err := ed25519.GenerateKey(rand.Reader); err != nil {
@@ -92,7 +130,6 @@ func (t *Tor) Serve() error {
92130
if err != nil {
93131
logger.Fatal("failed to start tor: %v", err)
94132
}
95-
defer e.Close()
96133
e.DeleteDataDirOnClose = true
97134
e.StopProcessOnClose = false
98135

@@ -102,40 +139,31 @@ func (t *Tor) Serve() error {
102139
if err != nil {
103140
logger.Fatal("failed to create onion service: %v", err)
104141
}
105-
defer onion.Close()
106142
onion.CloseLocalListenerOnClose = false
107143

108144
logger.Info(`listening on "%s" without TLS`, color.BlueString(onion.LocalListener.Addr().String()))
109145
logger.Info("please open a Tor capable browser and navigate to http://%v.onion", onion.ID)
110146

111-
server := http.Server{Handler: newWeb().handle(t.pool)}
112147
go func() {
113148
if err := server.Serve(onion); err != nil {
114-
logger.Error("serve tor hidden service failed: %v", err)
149+
logger.Fatal("serve tor hidden service failed: %v", err)
115150
}
116-
}()
117151

118-
<-t.ctx.Done()
119-
logger.Info("stopping tor hidden service...")
120-
if err := server.Shutdown(t.ctx); err != nil {
121-
logger.Error("shutdown tor hidden service failed: %v", err)
122-
return err
123-
}
124-
125-
return errors.New("done")
152+
// Close onion service.
153+
// TODO
154+
<-t.ctx.Done()
155+
e.Close()
156+
onion.Close()
157+
}()
126158
}
127159

128-
func (t *Tor) torrc() string {
129-
if config.Opts.TorrcFile() == "" {
130-
return ""
131-
}
132-
if torPortBusy() {
133-
return ""
134-
}
135-
if _, err := os.Open(config.Opts.TorrcFile()); err != nil {
136-
return ""
137-
}
138-
return config.Opts.TorrcFile()
160+
func startHTTPServer(server *http.Server) {
161+
go func() {
162+
logger.Info(`Listening on "%s" without TLS`, color.BlueString(server.Addr))
163+
if err := server.ListenAndServe(); err != http.ErrServerClosed {
164+
logger.Fatal("Server failed to start: %v", err)
165+
}
166+
}()
139167
}
140168

141169
func torPortBusy() bool {
@@ -153,3 +181,10 @@ func torPortBusy() bool {
153181

154182
return false
155183
}
184+
185+
func torExist() bool {
186+
if _, err := exec.LookPath("tor"); err != nil {
187+
return false
188+
}
189+
return true
190+
}

‎wayback.1

+6
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ Log level, supported level are "debug", "info", "warn", "error", "fatal", defaul
123123
.B ENABLE_METRICS
124124
Enable metrics collector\&.
125125
.TP
126+
.B HTTP_LISTEN_ADDR
127+
The listen address for the HTTP server. default "127.0.0.1:8964"\&.
128+
.TP
129+
.B CHROME_REMOTE_ADDR
130+
Chrome/Chromium remote debugging address, for screenshot\&.
131+
.TP
126132
.B WAYBACK_IPFS_HOST
127133
IPFS daemon service's host, do not require, unless enable ipfs. default "127.0.0.1". (same as flag --ipfs-host)\&.
128134
.TP

‎wayback.conf

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ WAYBACK_TOR_PRIVKEY=
4343
WAYBACK_TOR_LOCAL_PORT=8964
4444
WAYBACK_TOR_REMOTE_PORTS=80
4545
WAYBACK_TORRC=/etc/tor/torrc
46+
HTTP_LISTEN_ADDR=127.0.0.1:8964
4647
CHROME_REMOTE_ADDR=
4748
WAYBACK_POOLING_SIZE=3
4849
WAYBACK_STORAGE_DIR=

0 commit comments

Comments
 (0)