Skip to content

Commit b7cb677

Browse files
wwqgtxxwwqgtxx
committed
chore: support ETag for update geo
1 parent 5d24251 commit b7cb677

File tree

11 files changed

+117
-51
lines changed

11 files changed

+117
-51
lines changed

‎adapter/provider/parser.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func ParseProxyProvider(name string, mapping map[string]any) (types.ProxyProvide
111111
return nil, fmt.Errorf("%w: %s", errSubPath, path)
112112
}
113113
}
114-
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header)
114+
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header, resource.DefaultHttpTimeout)
115115
default:
116116
return nil, fmt.Errorf("%w: %s", errVehicleType, schema.Type)
117117
}

‎component/resource/fetcher.go

+1-19
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package resource
33
import (
44
"context"
55
"os"
6-
"path/filepath"
76
"time"
87

98
types "github.com/metacubex/mihomo/constant/provider"
@@ -13,11 +12,6 @@ import (
1312
"github.com/samber/lo"
1413
)
1514

16-
var (
17-
fileMode os.FileMode = 0o666
18-
dirMode os.FileMode = 0o755
19-
)
20-
2115
type Parser[V any] func([]byte) (V, error)
2216

2317
type Fetcher[V any] struct {
@@ -118,7 +112,7 @@ func (f *Fetcher[V]) loadBuf(buf []byte, hash types.HashType, updateFile bool) (
118112
}
119113

120114
if updateFile {
121-
if err = safeWrite(f.vehicle.Path(), buf); err != nil {
115+
if err = f.vehicle.Write(buf); err != nil {
122116
return lo.Empty[V](), false, err
123117
}
124118
}
@@ -205,18 +199,6 @@ func (f *Fetcher[V]) updateWithLog() {
205199
return
206200
}
207201

208-
func safeWrite(path string, buf []byte) error {
209-
dir := filepath.Dir(path)
210-
211-
if _, err := os.Stat(dir); os.IsNotExist(err) {
212-
if err := os.MkdirAll(dir, dirMode); err != nil {
213-
return err
214-
}
215-
}
216-
217-
return os.WriteFile(path, buf, fileMode)
218-
}
219-
220202
func NewFetcher[V any](name string, interval time.Duration, vehicle types.Vehicle, parser Parser[V], onUpdate func(V)) *Fetcher[V] {
221203
ctx, cancel := context.WithCancel(context.Background())
222204
return &Fetcher[V]{

‎component/resource/vehicle.go

+40-10
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,33 @@ import (
66
"io"
77
"net/http"
88
"os"
9+
"path/filepath"
910
"time"
1011

1112
mihomoHttp "github.com/metacubex/mihomo/component/http"
1213
"github.com/metacubex/mihomo/component/profile/cachefile"
1314
types "github.com/metacubex/mihomo/constant/provider"
1415
)
1516

17+
const (
18+
DefaultHttpTimeout = time.Second * 20
19+
20+
fileMode os.FileMode = 0o666
21+
dirMode os.FileMode = 0o755
22+
)
23+
24+
func safeWrite(path string, buf []byte) error {
25+
dir := filepath.Dir(path)
26+
27+
if _, err := os.Stat(dir); os.IsNotExist(err) {
28+
if err := os.MkdirAll(dir, dirMode); err != nil {
29+
return err
30+
}
31+
}
32+
33+
return os.WriteFile(path, buf, fileMode)
34+
}
35+
1636
type FileVehicle struct {
1737
path string
1838
}
@@ -42,15 +62,20 @@ func (f *FileVehicle) Proxy() string {
4262
return ""
4363
}
4464

65+
func (f *FileVehicle) Write(buf []byte) error {
66+
return safeWrite(f.path, buf)
67+
}
68+
4569
func NewFileVehicle(path string) *FileVehicle {
4670
return &FileVehicle{path: path}
4771
}
4872

4973
type HTTPVehicle struct {
50-
url string
51-
path string
52-
proxy string
53-
header http.Header
74+
url string
75+
path string
76+
proxy string
77+
header http.Header
78+
timeout time.Duration
5479
}
5580

5681
func (h *HTTPVehicle) Url() string {
@@ -69,8 +94,12 @@ func (h *HTTPVehicle) Proxy() string {
6994
return h.proxy
7095
}
7196

97+
func (h *HTTPVehicle) Write(buf []byte) error {
98+
return safeWrite(h.path, buf)
99+
}
100+
72101
func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []byte, hash types.HashType, err error) {
73-
ctx, cancel := context.WithTimeout(ctx, time.Second*20)
102+
ctx, cancel := context.WithTimeout(ctx, h.timeout)
74103
defer cancel()
75104
header := h.header
76105
setIfNoneMatch := false
@@ -107,11 +136,12 @@ func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []b
107136
return
108137
}
109138

110-
func NewHTTPVehicle(url string, path string, proxy string, header http.Header) *HTTPVehicle {
139+
func NewHTTPVehicle(url string, path string, proxy string, header http.Header, timeout time.Duration) *HTTPVehicle {
111140
return &HTTPVehicle{
112-
url: url,
113-
path: path,
114-
proxy: proxy,
115-
header: header,
141+
url: url,
142+
path: path,
143+
proxy: proxy,
144+
header: header,
145+
timeout: timeout,
116146
}
117147
}

‎component/updater/update_geo.go

+63-12
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ import (
1313
"github.com/metacubex/mihomo/component/geodata"
1414
_ "github.com/metacubex/mihomo/component/geodata/standard"
1515
"github.com/metacubex/mihomo/component/mmdb"
16+
"github.com/metacubex/mihomo/component/resource"
1617
C "github.com/metacubex/mihomo/constant"
18+
P "github.com/metacubex/mihomo/constant/provider"
1719
"github.com/metacubex/mihomo/log"
1820

1921
"github.com/oschwald/maxminddb-golang"
@@ -43,73 +45,122 @@ func SetGeoUpdateInterval(newGeoUpdateInterval int) {
4345
}
4446

4547
func UpdateMMDB() (err error) {
46-
defer mmdb.ReloadIP()
47-
data, err := downloadForBytes(geodata.MmdbUrl())
48+
vehicle := resource.NewHTTPVehicle(geodata.MmdbUrl(), C.Path.MMDB(), "", nil, defaultHttpTimeout)
49+
var oldHash P.HashType
50+
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
51+
oldHash = P.MakeHash(buf)
52+
}
53+
data, hash, err := vehicle.Read(context.Background(), oldHash)
4854
if err != nil {
4955
return fmt.Errorf("can't download MMDB database file: %w", err)
5056
}
57+
if oldHash.Equal(hash) { // same hash, ignored
58+
return nil
59+
}
60+
if len(data) == 0 {
61+
return fmt.Errorf("can't download MMDB database file: no data")
62+
}
63+
5164
instance, err := maxminddb.FromBytes(data)
5265
if err != nil {
5366
return fmt.Errorf("invalid MMDB database file: %s", err)
5467
}
5568
_ = instance.Close()
5669

70+
defer mmdb.ReloadIP()
5771
mmdb.IPInstance().Reader.Close() // mmdb is loaded with mmap, so it needs to be closed before overwriting the file
58-
if err = saveFile(data, C.Path.MMDB()); err != nil {
72+
if err = vehicle.Write(data); err != nil {
5973
return fmt.Errorf("can't save MMDB database file: %w", err)
6074
}
6175
return nil
6276
}
6377

6478
func UpdateASN() (err error) {
65-
defer mmdb.ReloadASN()
66-
data, err := downloadForBytes(geodata.ASNUrl())
79+
vehicle := resource.NewHTTPVehicle(geodata.ASNUrl(), C.Path.ASN(), "", nil, defaultHttpTimeout)
80+
var oldHash P.HashType
81+
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
82+
oldHash = P.MakeHash(buf)
83+
}
84+
data, hash, err := vehicle.Read(context.Background(), oldHash)
6785
if err != nil {
6886
return fmt.Errorf("can't download ASN database file: %w", err)
6987
}
88+
if oldHash.Equal(hash) { // same hash, ignored
89+
return nil
90+
}
91+
if len(data) == 0 {
92+
return fmt.Errorf("can't download ASN database file: no data")
93+
}
7094

7195
instance, err := maxminddb.FromBytes(data)
7296
if err != nil {
7397
return fmt.Errorf("invalid ASN database file: %s", err)
7498
}
7599
_ = instance.Close()
76100

101+
defer mmdb.ReloadASN()
77102
mmdb.ASNInstance().Reader.Close() // mmdb is loaded with mmap, so it needs to be closed before overwriting the file
78-
if err = saveFile(data, C.Path.ASN()); err != nil {
103+
if err = vehicle.Write(data); err != nil {
79104
return fmt.Errorf("can't save ASN database file: %w", err)
80105
}
81106
return nil
82107
}
83108

84109
func UpdateGeoIp() (err error) {
85-
defer geodata.ClearGeoIPCache()
86110
geoLoader, err := geodata.GetGeoDataLoader("standard")
87-
data, err := downloadForBytes(geodata.GeoIpUrl())
111+
112+
vehicle := resource.NewHTTPVehicle(geodata.GeoIpUrl(), C.Path.GeoIP(), "", nil, defaultHttpTimeout)
113+
var oldHash P.HashType
114+
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
115+
oldHash = P.MakeHash(buf)
116+
}
117+
data, hash, err := vehicle.Read(context.Background(), oldHash)
88118
if err != nil {
89119
return fmt.Errorf("can't download GeoIP database file: %w", err)
90120
}
121+
if oldHash.Equal(hash) { // same hash, ignored
122+
return nil
123+
}
124+
if len(data) == 0 {
125+
return fmt.Errorf("can't download GeoIP database file: no data")
126+
}
127+
91128
if _, err = geoLoader.LoadIPByBytes(data, "cn"); err != nil {
92129
return fmt.Errorf("invalid GeoIP database file: %s", err)
93130
}
94-
if err = saveFile(data, C.Path.GeoIP()); err != nil {
131+
132+
defer geodata.ClearGeoIPCache()
133+
if err = vehicle.Write(data); err != nil {
95134
return fmt.Errorf("can't save GeoIP database file: %w", err)
96135
}
97136
return nil
98137
}
99138

100139
func UpdateGeoSite() (err error) {
101-
defer geodata.ClearGeoSiteCache()
102140
geoLoader, err := geodata.GetGeoDataLoader("standard")
103-
data, err := downloadForBytes(geodata.GeoSiteUrl())
141+
142+
vehicle := resource.NewHTTPVehicle(geodata.GeoSiteUrl(), C.Path.GeoSite(), "", nil, defaultHttpTimeout)
143+
var oldHash P.HashType
144+
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
145+
oldHash = P.MakeHash(buf)
146+
}
147+
data, hash, err := vehicle.Read(context.Background(), oldHash)
104148
if err != nil {
105149
return fmt.Errorf("can't download GeoSite database file: %w", err)
106150
}
151+
if oldHash.Equal(hash) { // same hash, ignored
152+
return nil
153+
}
154+
if len(data) == 0 {
155+
return fmt.Errorf("can't download GeoSite database file: no data")
156+
}
107157

108158
if _, err = geoLoader.LoadSiteByBytes(data, "cn"); err != nil {
109159
return fmt.Errorf("invalid GeoSite database file: %s", err)
110160
}
111161

112-
if err = saveFile(data, C.Path.GeoSite()); err != nil {
162+
defer geodata.ClearGeoSiteCache()
163+
if err = vehicle.Write(data); err != nil {
113164
return fmt.Errorf("can't save GeoSite database file: %w", err)
114165
}
115166
return nil

‎component/updater/update_ui.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@ import (
1717
var (
1818
ExternalUIURL string
1919
ExternalUIPath string
20-
AutoUpdateUI bool
20+
AutoDownloadUI bool
2121
)
2222

2323
var xdMutex sync.Mutex
2424

25-
func UpdateUI() error {
25+
func DownloadUI() error {
2626
xdMutex.Lock()
2727
defer xdMutex.Unlock()
2828

‎component/updater/utils.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ import (
1313
"golang.org/x/exp/constraints"
1414
)
1515

16+
const defaultHttpTimeout = time.Second * 90
17+
1618
func downloadForBytes(url string) ([]byte, error) {
17-
ctx, cancel := context.WithTimeout(context.Background(), time.Second*90)
19+
ctx, cancel := context.WithTimeout(context.Background(), defaultHttpTimeout)
1820
defer cancel()
1921
resp, err := mihomoHttp.HttpRequest(ctx, url, http.MethodGet, nil, nil)
2022
if err != nil {

‎config/config.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -701,7 +701,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) {
701701

702702
// checkout externalUI exist
703703
if cfg.ExternalUI != "" {
704-
updater.AutoUpdateUI = true
704+
updater.AutoDownloadUI = true
705705
updater.ExternalUIPath = C.Path.Resolve(cfg.ExternalUI)
706706
} else {
707707
// default externalUI path
@@ -710,7 +710,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) {
710710

711711
// checkout UIpath/name exist
712712
if cfg.ExternalUIName != "" {
713-
updater.AutoUpdateUI = true
713+
updater.AutoDownloadUI = true
714714
updater.ExternalUIPath = path.Join(updater.ExternalUIPath, cfg.ExternalUIName)
715715
}
716716

‎constant/provider/interface.go

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ func (v VehicleType) String() string {
3333

3434
type Vehicle interface {
3535
Read(ctx context.Context, oldHash HashType) (buf []byte, hash HashType, err error)
36+
Write(buf []byte) error
3637
Path() string
3738
Url() string
3839
Proxy() string

‎hub/executor/executor.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -381,13 +381,13 @@ func updateTunnels(tunnels []LC.Tunnel) {
381381
}
382382

383383
func initExternalUI() {
384-
if updater.AutoUpdateUI {
384+
if updater.AutoDownloadUI {
385385
dirEntries, _ := os.ReadDir(updater.ExternalUIPath)
386386
if len(dirEntries) > 0 {
387387
log.Infoln("UI already exists, skip downloading")
388388
} else {
389389
log.Infoln("External UI downloading ...")
390-
updater.UpdateUI()
390+
updater.DownloadUI()
391391
}
392392
}
393393
}

‎hub/route/upgrade.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func upgradeCore(w http.ResponseWriter, r *http.Request) {
4747
}
4848

4949
func updateUI(w http.ResponseWriter, r *http.Request) {
50-
err := updater.UpdateUI()
50+
err := updater.DownloadUI()
5151
if err != nil {
5252
log.Warnln("%s", err)
5353
render.Status(r, http.StatusInternalServerError)

‎rules/provider/parse.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func ParseRuleProvider(name string, mapping map[string]interface{}, parse func(t
5353
return nil, fmt.Errorf("%w: %s", errSubPath, path)
5454
}
5555
}
56-
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil)
56+
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil, resource.DefaultHttpTimeout)
5757
default:
5858
return nil, fmt.Errorf("unsupported vehicle type: %s", schema.Type)
5959
}

0 commit comments

Comments
 (0)