Skip to content

Commit 17730df

Browse files
hktalenthktalent
committed
up 2022-08-23
1 parent 250afe2 commit 17730df

File tree

107 files changed

+15353
-59
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

107 files changed

+15353
-59
lines changed

‎.github/build/linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ builds:
1616
- linux
1717
goarch:
1818
- amd64
19+
- arm64
1920
archives:
2021
- format: zip
2122

‎.github/build/mac.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ builds:
1616
- darwin
1717
goarch:
1818
- amd64
19+
- arm64
1920

2021
archives:
2122
- format: zip

‎.github/build/mac_arm64.yml‎

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎.github/build/windows.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ env:
33
before:
44
hooks:
55
- go mod tidy
6+
#after:
7+
# hooks:
8+
# post: ./upx.sh
69
project_name: scan4all
710
builds:
811
- id: scan4all-windows

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,6 @@ on:
66
workflow_dispatch:
77

88
jobs:
9-
build-mac-arm:
10-
runs-on: macos-latest
11-
steps:
12-
- name: Code checkout
13-
uses: actions/checkout@v2
14-
with:
15-
fetch-depth: 0
16-
- name: Set up Go
17-
uses: actions/setup-go@v2
18-
with:
19-
go-version: 1.18
20-
- name: Install Dependences
21-
run: brew install libpcap
22-
- name: Run GoReleaser
23-
uses: goreleaser/goreleaser-action@v2
24-
with:
25-
version: latest
26-
args: release -f .github/build/mac_arm64.yml --rm-dist
27-
env:
28-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
299
build-mac:
3010
runs-on: macos-latest
3111
steps:
@@ -38,7 +18,7 @@ jobs:
3818
with:
3919
go-version: 1.18
4020
- name: Install Dependences
41-
run: brew install libpcap
21+
run: brew install libpcap upx
4222
- name: Run GoReleaser
4323
uses: goreleaser/goreleaser-action@v2
4424
with:
@@ -59,7 +39,7 @@ jobs:
5939
with:
6040
go-version: 1.18
6141
- name: Install Dependences
62-
run: sudo apt install -yy libpcap-dev
42+
run: sudo apt install -yy libpcap-dev upx
6343

6444
- name: Run GoReleaser
6545
uses: goreleaser/goreleaser-action@v2

‎.github/workflows/upx.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/usr/bin/env bash
2+
3+
for FILE in dist/scan4all_*/*; do
4+
du -sh ${FILE}
5+
upx ${FILE}
6+
du -sh ${FILE}
7+
done

‎go.mod‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ require github.com/spaolacci/murmur3 v1.1.0
7777

7878
require (
7979
github.com/PuerkitoBio/goquery v1.8.0
80+
github.com/Qianlitp/crawlergo v0.4.3
8081
github.com/RumbleDiscovery/jarm-go v0.0.6
8182
github.com/Ullaakut/nmap v2.0.2+incompatible
8283
github.com/ammario/ipisp/v2 v2.0.0
@@ -120,6 +121,7 @@ require (
120121
github.com/shirou/gopsutil/v3 v3.22.7
121122
github.com/sijms/go-ora/v2 v2.4.25
122123
github.com/simonnilsson/ask v0.2.0
124+
github.com/sirupsen/logrus v1.7.0
123125
github.com/spf13/viper v1.12.0
124126
github.com/stacktitan/smb v0.0.0-20190531122847-da9a425dceb8
125127
github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible
@@ -160,6 +162,7 @@ require (
160162
github.com/cespare/xxhash/v2 v2.1.2 // indirect
161163
github.com/chromedp/sysutil v1.0.0 // indirect
162164
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
165+
github.com/deckarep/golang-set v1.7.1 // indirect
163166
github.com/dgraph-io/ristretto v0.1.0 // indirect
164167
github.com/dimchansky/utfbom v1.1.1 // indirect
165168
github.com/docker/go-units v0.4.0 // indirect
@@ -178,6 +181,7 @@ require (
178181
github.com/gobwas/pool v0.2.1 // indirect
179182
github.com/gobwas/ws v1.1.0 // indirect
180183
github.com/gofrs/uuid v4.2.0+incompatible // indirect
184+
github.com/gogf/gf v1.16.6 // indirect
181185
github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
182186
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
183187
github.com/golang-sql/sqlexp v0.1.0 // indirect
@@ -232,6 +236,7 @@ require (
232236
github.com/nwaples/rardecode v1.1.2 // indirect
233237
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 // indirect
234238
github.com/owenrumney/go-sarif/v2 v2.1.2 // indirect
239+
github.com/panjf2000/ants/v2 v2.2.2 // indirect
235240
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
236241
github.com/pelletier/go-toml v1.9.5 // indirect
237242
github.com/pelletier/go-toml/v2 v2.0.1 // indirect

‎go.sum‎

Lines changed: 35 additions & 0 deletions
Large diffs are not rendered by default.

‎pkg/fingerprint/fgConst.go‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
_ "embed"
66
"encoding/json"
77
"fmt"
8+
"github.com/hktalent/scan4all/lib/util"
89
"io/ioutil"
910
"log"
1011
"net/http"
@@ -39,6 +40,8 @@ var FgType map[int]string = map[int]string{
3940

4041
//go:embed dicts/fg.json
4142
var FgData string
43+
44+
// 指纹 {id：指纹数据对象}
4245
var FGDataMap []map[string]interface{}
4346

4447
func Get4K(m *map[string]interface{}, k string) string {
@@ -57,11 +60,11 @@ func MergeReqUrl() {
5760
LoadWebfingerprintEhole()
5861
x1 := GetWebfingerprintEhole()
5962
// 测试的时候下面代码才打开
60-
//if "true" == pkg.GetValByDefault("MyDebug", "false") {
61-
// x1.Fingerprint = []*Fingerprint{}
62-
// localFinger = "{}"
63-
// log.Println("MyDebug")
64-
//}
63+
if "true" == util.GetValByDefault("MyDebug", "false") {
64+
x1.Fingerprint = []*Fingerprint{}
65+
localFinger = "{}"
66+
log.Println("MyDebug")
67+
}
6568

6669
// 不重复的URL
6770
var urls = []string{}
@@ -134,6 +137,13 @@ func DelTmpFgFile() {
134137
// 这里可以动态加载远程的url指纹数据到 FgData
135138
func init() {
136139
json.Unmarshal([]byte(FgData), &FGDataMap)
140+
var aN []map[string]interface{}
141+
for _, x := range FGDataMap {
142+
if bD, ok := x["delete"]; ok && false == bD.(bool) {
143+
aN = append(aN, x)
144+
}
145+
}
146+
FGDataMap = aN
137147
MergeReqUrl()
138148
var err error
139149
tempInput1, err = ioutil.TempFile("", "dict-in-*")

‎pkg/fingerprint/fingerScan.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package fingerprint
33
import (
44
"encoding/hex"
55
"encoding/json"
6+
"fmt"
67
"github.com/hktalent/scan4all/lib/util"
78
"log"
89
"net/url"
@@ -50,7 +51,7 @@ var MUrl *sync.Map = new(sync.Map)
5051
// 图标每个目标只识别一次
5152
var Mfavhash *sync.Map = new(sync.Map)
5253

53-
// 一个url到底和多少组件id关联
54+
// 一个url到底和多少组件id关联
5455
var MFid *sync.Map = new(sync.Map)
5556

5657
// 清除数据
@@ -149,10 +150,10 @@ func CaseMethod(szUrl, method, bodyString, favhash, md5Body, hexBody string, fin
149150
var enableFingerTitleHeaderMd5Hex = util.GetValAsBool("enableFingerTitleHeaderMd5Hex")
150151

151152
// 相同的url、组件（产品），>=2 个指纹命中，那么该组件的其他指纹匹配将跳过
152-
func FingerScan(headers map[string][]string, body []byte, title string, url string, status_code string) []string {
153+
func FingerScan(headers map[string][]string, body []byte, title string, url string, status_code string) ([]string, []string) {
153154
if nil == body || 0 == len(body) {
154155
//log.Println(url, " 存在异常，body为nil")
155-
return []string{}
156+
return []string{}, nil
156157
}
157158
//log.Println("FgDictFile = ", FgDictFile)
158159
bodyString := string(body)
@@ -174,8 +175,10 @@ func FingerScan(headers map[string][]string, body []byte, title string, url stri
174175
}
175176

176177
var cms []string
178+
var fgIds []string
177179
for _, x1 := range []*Packjson{EholeFinpx, LocalFinpx} {
178180
for _, finp := range x1.Fingerprint {
181+
n1 := len(cms)
179182
if finp.UrlPath == "" || strings.HasSuffix(url, finp.UrlPath) {
180183
//if -1 < strings.Index(url, "/favicon.ico") && finp.Cms == "SpringBoot" {
181184
// log.Println(url)
@@ -195,7 +198,14 @@ func FingerScan(headers map[string][]string, body []byte, title string, url stri
195198
}
196199
}
197200
}
201+
// 找到指纹
202+
if len(cms) > n1 {
203+
fgIds = append(fgIds, fmt.Sprintf("%v", finp.Id))
204+
log.Printf("%d\n", finp.Id)
205+
n1 = len(cms)
206+
}
198207
}
208+
199209
}
200-
return cms
210+
return cms, fgIds
201211
}

0 commit comments

Comments
 (0)