Skip to content

Commit 6539ac7

Browse files
kappajkappaj
committed
Update output file on each result line
1 parent 15ea2bc commit 6539ac7

File tree

3 files changed

+159
-84
lines changed

3 files changed

+159
-84
lines changed

‎internal/grawl/file_writer.go‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package grawl
2+
3+
import (
4+
"encoding/csv"
5+
"fmt"
6+
"io"
7+
"os"
8+
"strconv"
9+
"sync"
10+
)
11+
12+
type FileWriter struct {
13+
sync.RWMutex
14+
filePath string
15+
fileInitialized bool
16+
}
17+
18+
func NewFileWriter(filePath string) *FileWriter {
19+
return &FileWriter{
20+
filePath: filePath,
21+
fileInitialized: false,
22+
}
23+
}
24+
25+
func (f *FileWriter) InitFile() {
26+
if f.fileInitialized {
27+
return
28+
}
29+
30+
fmt.Printf("Saving file \"%s\".\n", f.filePath)
31+
32+
file, err := os.Create(f.filePath)
33+
if err != nil {
34+
panic(err)
35+
}
36+
defer file.Close()
37+
38+
writer := csv.NewWriter(file)
39+
writer.Comma = ';'
40+
defer writer.Flush()
41+
42+
headers := f.getCsvHeader()
43+
f.write(headers, file)
44+
f.fileInitialized = true
45+
}
46+
47+
func (f *FileWriter) WriteResultLine(r *Result) {
48+
f.RLock()
49+
defer f.RUnlock()
50+
51+
if !f.fileInitialized {
52+
panic("csv not initialized yet")
53+
}
54+
55+
file, err := os.OpenFile(f.filePath, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
56+
if err != nil {
57+
panic(err)
58+
}
59+
defer file.Close()
60+
61+
line := f.getCsvRow(r)
62+
f.write(line, file)
63+
}
64+
65+
func (f *FileWriter) getCsvRow(r *Result) []string {
66+
67+
errorText := ""
68+
if r.error != nil {
69+
errorText = fmt.Sprintf("%v", r.error)
70+
}
71+
72+
return []string{
73+
r.responseAt.Format(DateFormat),
74+
strconv.Itoa(r.statusCode),
75+
r.status,
76+
r.url,
77+
78+
r.foundOnUrl,
79+
r.contentType,
80+
strconv.FormatInt(r.duration.Milliseconds(), 10),
81+
strconv.Itoa(r.depth),
82+
r.urlRedirectedFrom,
83+
84+
r.urlHost,
85+
r.urlPath,
86+
r.urlParmeters,
87+
r.urlFragment,
88+
89+
errorText,
90+
}
91+
}
92+
93+
func (f *FileWriter) getCsvHeader() []string {
94+
return []string{
95+
"Response time",
96+
"Status code",
97+
"Status",
98+
"URL",
99+
100+
"Found on URL",
101+
"Content type",
102+
"Duration (ms)",
103+
"Depth",
104+
"Redirected from",
105+
106+
"Host",
107+
"Path",
108+
"Parameters",
109+
"Fragment",
110+
111+
"Info / error",
112+
}
113+
}
114+
115+
func (f *FileWriter) write(text []string, file io.Writer) {
116+
writer := csv.NewWriter(file)
117+
writer.Comma = ';'
118+
defer writer.Flush()
119+
120+
if err := writer.Write(text); err != nil {
121+
panic(err)
122+
}
123+
}

‎internal/grawl/grawler.go‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,12 @@ package grawl
22

33
import (
44
"encoding/base64"
5-
"encoding/csv"
65
"errors"
76
"fmt"
87
"github.com/fatih/color"
98
"github.com/gocolly/colly/v2"
109
"github.com/manifoldco/promptui"
1110
url2 "net/url"
12-
"os"
1311
"regexp"
1412
"slices"
1513
"strings"
@@ -29,6 +27,7 @@ type Grawler struct {
2927
errorCount uint32
3028
totalDuration time.Duration
3129
runningRequests *RunningRequests
30+
fileWriter *FileWriter
3231
}
3332

3433
func NewGrawler(flags Flags) *Grawler {
@@ -157,6 +156,10 @@ func (g *Grawler) Grawl(url string) {
157156

158157
reqResult.UpdateOnResponse(r, g.responseCount, duration, nil)
159158
g.printResult(reqResult)
159+
160+
if g.fileWriter != nil {
161+
g.fileWriter.WriteResultLine(reqResult)
162+
}
160163
} else {
161164
fmt.Printf("No start time found for %s\n", r.Request.URL)
162165
}
@@ -187,7 +190,6 @@ func (g *Grawler) Grawl(url string) {
187190
reqResult.UpdateOnResponse(r, g.responseCount, duration, &err)
188191
g.totalDuration += duration
189192
g.printResult(reqResult)
190-
//fmt.Println("error:", err)
191193
} else {
192194
fmt.Println("Request data not found", r.Request.URL)
193195
}
@@ -255,16 +257,17 @@ func (g *Grawler) Grawl(url string) {
255257
})
256258
}
257259

260+
if g.flags.FlagOutputFilename != "" {
261+
g.fileWriter = NewFileWriter(g.flags.FlagOutputFilename)
262+
g.fileWriter.InitFile()
263+
}
264+
258265
err = c.Visit(url)
259266
if err != nil {
260267
fmt.Printf("Could not visit url: %v\n", err)
261268
return
262269
}
263270

264-
if g.flags.FlagOutputFilename != "" {
265-
g.saveCsvFile(g.runningRequests)
266-
}
267-
268271
g.printSummary()
269272
}
270273

@@ -302,32 +305,32 @@ func (g *Grawler) printSummary() {
302305
fmt.Println("Errors/Skipped: ", g.errorCount)
303306
}
304307

305-
func (g *Grawler) saveCsvFile(runningRequests *RunningRequests) {
306-
fmt.Printf("Saving file \"%s\".\n", g.flags.FlagOutputFilename)
307-
308-
results := runningRequests.GetValues()
309-
310-
file, err := os.Create(g.flags.FlagOutputFilename)
311-
if err != nil {
312-
panic(err)
313-
}
314-
defer file.Close()
315-
316-
writer := csv.NewWriter(file)
317-
writer.Comma = ';'
318-
defer writer.Flush()
319-
320-
headers := GetCsvHeader()
321-
if err := writer.Write(headers); err != nil {
322-
panic(err)
323-
}
324-
325-
for _, result := range *results {
326-
if err := writer.Write(result.GetCsvRow()); err != nil {
327-
panic(err)
328-
}
329-
}
330-
}
308+
//func (g *Grawler) saveCsvFile(runningRequests *RunningRequests) {
309+
// fmt.Printf("Saving file \"%s\".\n", g.flags.FlagOutputFilename)
310+
//
311+
// results := runningRequests.GetValues()
312+
//
313+
// file, err := os.Create(g.flags.FlagOutputFilename)
314+
// if err != nil {
315+
// panic(err)
316+
// }
317+
// defer file.Close()
318+
//
319+
// writer := csv.NewWriter(file)
320+
// writer.Comma = ';'
321+
// defer writer.Flush()
322+
//
323+
// headers := GetCsvHeader()
324+
// if err := writer.Write(headers); err != nil {
325+
// panic(err)
326+
// }
327+
//
328+
// for _, result := range *results {
329+
// if err := writer.Write(result.GetCsvRow()); err != nil {
330+
// panic(err)
331+
// }
332+
// }
333+
//}
331334

332335
func (g *Grawler) promptPassword() (string, error) {
333336
validate := func(input string) error {

‎internal/grawl/request_result.go‎

Lines changed: 0 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package grawl
22

33
import (
4-
"fmt"
54
"github.com/gocolly/colly/v2"
65
"net/http"
76
"strconv"
@@ -108,56 +107,6 @@ func (r *Result) GetPrintRow() string {
108107

109108
}
110109

111-
func GetCsvHeader() []string {
112-
return []string{
113-
"Response time",
114-
"Status code",
115-
"Status",
116-
"URL",
117-
118-
"Found on URL",
119-
"Content type",
120-
"Duration (ms)",
121-
"Depth",
122-
"Redirected from",
123-
124-
"Host",
125-
"Path",
126-
"Parameters",
127-
"Fragment",
128-
129-
"Info / error",
130-
}
131-
}
132-
133-
func (r *Result) GetCsvRow() []string {
134-
135-
errorText := ""
136-
if r.error != nil {
137-
errorText = fmt.Sprintf("%v", r.error)
138-
}
139-
140-
return []string{
141-
r.responseAt.Format(DateFormat),
142-
strconv.Itoa(r.statusCode),
143-
r.status,
144-
r.url,
145-
146-
r.foundOnUrl,
147-
r.contentType,
148-
strconv.FormatInt(r.duration.Milliseconds(), 10),
149-
strconv.Itoa(r.depth),
150-
r.urlRedirectedFrom,
151-
152-
r.urlHost,
153-
r.urlPath,
154-
r.urlParmeters,
155-
r.urlFragment,
156-
157-
errorText,
158-
}
159-
}
160-
161110
func (r *Result) HasError() bool {
162111
return r.error != nil || r.statusCode >= 400
163112
}

0 commit comments

Comments
 (0)