Skip to content

Commit aacdb12

Browse files
author
Zef Hemel
committed
README
1 parent e23c71f commit aacdb12

File tree

5 files changed

+210
-71
lines changed

5 files changed

+210
-71
lines changed

‎README.md‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# Ax
2+
It's a structured logging world we live in, but do we really have to look at JSON? Not with Ax.
3+
4+
## Installation
5+
For now there's no pre-built binaries, so to run this you need a reasonably recent version of Go. Then either git clone this project in `$GOPATH/src/github.com/zefhemel/ax` or run `go get -u github.com/zefhemel/ax`.
6+
7+
To install dependencies:
8+
9+
make dep
10+
11+
To run tests:
12+
13+
make test
14+
15+
To "go install" ax (this will put the resulting binary in `$GOPATH/bin` so put that in your `$PATH`)
16+
17+
make
18+
19+
20+
## Upgrade
21+
22+
In `$GOPATH/src/github.com/zefhemel/ax`:
23+
24+
git pull
25+
make
26+
27+
## Setup
28+
Once you have `ax` installed, the first thing you'll want to do is setup bash or zsh command completion (I'm not kidding).
29+
30+
For bash, add to `~/.bash_profile`:
31+
32+
eval "$(ax --completion-script-bash)"
33+
34+
For zsh, add to `~/.zshrc`:
35+
36+
eval "$(ax --completion-script-zsh)"
37+
38+
After this, you can auto complete commands, flags, environments and even attribute names with TAB. Use it, love it.
39+
40+
## Setup with Kibana
41+
To setup Ax for use with Kibana, run:
42+
43+
ax env add
44+
45+
This will prompt you for a name, backend-type (kibana in this case), URL and if this URL is basic auth protected a username and password, and then an index.
46+
47+
To see if it works, just run:
48+
49+
ax --env yourenvname
50+
51+
Or, most likely your new env is the default (check with `ax env`) and you can just run:
52+
53+
ax
54+
55+
This should show you the (200) most recent logs.
56+
57+
If you're comfortable with YAML, you can run `ax env edit` which will open an editor with the `~/.config/ax/ax.yaml` file (either the editor set in your `EDITOR` env variable, with a fallback to `nano`). In there you can easily create more environments quickly.
58+
59+
## Use with Docker
60+
To use Ax with docker, simply use the `--docker` flag and a container name pattern. I usually use auto complete here (which works for docker containers too):
61+
62+
ax --docker turbo_
63+
64+
To query logs for all containers with "turbo\_" in the name. This assumes you have the `docker` binary in your path and setup properly.
65+
66+
## Use with log files or processes
67+
You can also pipe logs directly into Ax:
68+
69+
tail -f /var/log/something.log | ax
70+
71+
# Filtering and selecting attributes
72+
Looking at all logs is nice, but it only gets really interesting if you can start to filter stuff and by selecting only certain attributes.
73+
74+
To search for all logs containing the phrase "Traceback":
75+
76+
ax "Traceback"
77+
78+
To search for all logs with the phrase "Traceback" and where the attribute "domain" is set to "zef":
79+
80+
ax --where domain=zef "Traceback"
81+
82+
Again, after running Ax once on an environment it will cache attribute names, so you get completion for those too, usually.
83+
84+
Ax also supports the `!=` operator:
85+
86+
ax --where domain!=zef
87+
88+
If you have a lot of extra attributes in your log messages, you can select just a few of them:
89+
90+
ax --where domain=zef --select message --select tag
91+
92+
# "Tailing" logs
93+
Use the `-f` flag:
94+
95+
ax -f --where domain=zef
96+
97+
# Different output formats
98+
Don't like the default textual output, perhaps you prefer YAML:
99+
100+
ax --output yaml
101+
102+
or pretty JSON:
103+
104+
ax --output pretty-json
105+
106+
# Getting help
107+
108+
ax --help
109+
ax query --help

‎cmd/ax/query.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"fmt"
66
"os"
7+
"regexp"
78
"strings"
89
"time"
910

@@ -57,17 +58,21 @@ func selectHintAction() []string {
5758
return resultList
5859
}
5960

61+
var filterRegex = regexp.MustCompile(`([^!=<>]+)\s*(=|!=)\s*(.*)`)
62+
6063
func buildFilters(wheres []string) []common.QueryFilter {
6164
filters := make([]common.QueryFilter, 0, len(wheres))
6265
for _, whereClause := range wheres {
63-
pieces := strings.SplitN(whereClause, "=", 2)
64-
if len(pieces) != 2 {
66+
//pieces := strings.SplitN(whereClause, "=", 2)
67+
matches := filterRegex.FindAllStringSubmatch(whereClause, -1)
68+
if len(matches) != 1 {
6569
fmt.Println("Invalid where clause", whereClause)
6670
os.Exit(1)
6771
}
6872
filters = append(filters, common.QueryFilter{
69-
FieldName: pieces[0],
70-
Value: pieces[1],
73+
FieldName: matches[0][1],
74+
Operator: matches[0][2],
75+
Value: matches[0][3],
7176
})
7277
}
7378
return filters
@@ -126,10 +131,10 @@ func printMessage(message common.LogMessage, queryOutputFormat string) {
126131
fmt.Printf("%s ", messageColor.Sprint(msg))
127132
}
128133
for key, value := range message.Attributes {
129-
if key == "message" {
134+
if key == "message" || value == nil {
130135
continue
131136
}
132-
fmt.Printf("%s=%s ", color.CyanString(key), common.MustJsonEncode(value))
137+
fmt.Printf("%s=%+v ", color.CyanString(key), value)
133138
}
134139
fmt.Println()
135140
case "json":

‎pkg/backend/common/client.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type Client interface {
1616

1717
type QueryFilter struct {
1818
FieldName string
19+
Operator string
1920
Value string
2021
}
2122

@@ -126,7 +127,14 @@ func Project(m map[string]interface{}, fields []string) map[string]interface{} {
126127

127128
func (f QueryFilter) Matches(m LogMessage) bool {
128129
val, ok := m.Attributes[f.FieldName]
129-
return ok && f.Value == fmt.Sprintf("%v", val)
130+
switch f.Operator {
131+
case "=":
132+
return ok && f.Value == fmt.Sprintf("%v", val)
133+
case "!=":
134+
return !ok || (ok && f.Value != fmt.Sprintf("%v", val))
135+
default:
136+
panic("Not supported operator")
137+
}
130138
}
131139

132140
func matchesPhrase(s, phrase string) bool {

‎pkg/backend/common/client_test.go‎

Lines changed: 52 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -16,64 +16,67 @@ func TestFilter(t *testing.T) {
1616
}
1717
lastHour := time.Now().Add(-time.Hour)
1818
nextHour := time.Now().Add(time.Hour)
19-
shouldMatchQuery := Query{
20-
Filters: []QueryFilter{
21-
QueryFilter{FieldName: "someStr", Value: "Zef"},
19+
shouldMatchQueries := []Query{
20+
Query{
21+
Filters: []QueryFilter{
22+
QueryFilter{FieldName: "someStr", Value: "Zef", Operator: "="},
23+
},
2224
},
23-
}
24-
shouldMatchQuery2 := Query{
25-
Filters: []QueryFilter{
26-
QueryFilter{FieldName: "someN", Value: "34"},
25+
Query{
26+
Filters: []QueryFilter{
27+
QueryFilter{FieldName: "someN", Value: "34", Operator: "="},
28+
},
2729
},
28-
}
29-
shouldMatchQuery3 := Query{
30-
QueryString: "zef",
31-
Filters: []QueryFilter{
32-
QueryFilter{FieldName: "someN", Value: "34"},
30+
Query{
31+
QueryString: "zef",
32+
Filters: []QueryFilter{
33+
QueryFilter{FieldName: "someN", Value: "34", Operator: "="},
34+
},
3335
},
34-
}
35-
shouldMatchQuery4 := Query{
36-
QueryString: "zef",
37-
Filters: []QueryFilter{
38-
QueryFilter{FieldName: "someN", Value: "34"},
36+
Query{
37+
QueryString: "zef",
38+
Filters: []QueryFilter{
39+
QueryFilter{FieldName: "someN", Value: "34", Operator: "="},
40+
},
41+
Before: &nextHour,
42+
After: &lastHour,
3943
},
40-
Before: &nextHour,
41-
After: &lastHour,
42-
}
43-
shouldNotMatchQuery := Query{
44-
Filters: []QueryFilter{
45-
QueryFilter{FieldName: "someStr", Value: "Pete"},
44+
Query{
45+
Filters: []QueryFilter{
46+
QueryFilter{FieldName: "someN", Value: "32", Operator: "!="},
47+
},
4648
},
47-
}
48-
shouldNotMatchQuery2 := Query{
49-
QueryString: "bla",
50-
Filters: []QueryFilter{
51-
QueryFilter{FieldName: "someStr", Value: "Pete"},
49+
Query{
50+
Filters: []QueryFilter{
51+
QueryFilter{FieldName: "someNonexistingField", Value: "Pete", Operator: "!="},
52+
},
5253
},
5354
}
54-
shouldNotMatchQuery3 := Query{
55-
After: &nextHour,
56-
}
57-
if !MatchesQuery(lm, shouldMatchQuery) {
58-
t.Errorf("Did not match")
59-
}
60-
if !MatchesQuery(lm, shouldMatchQuery2) {
61-
t.Errorf("Did not match 2")
62-
}
63-
if !MatchesQuery(lm, shouldMatchQuery3) {
64-
t.Errorf("Did not match 3")
65-
}
66-
if !MatchesQuery(lm, shouldMatchQuery4) {
67-
t.Errorf("Did not match 4")
68-
}
69-
if MatchesQuery(lm, shouldNotMatchQuery) {
70-
t.Errorf("Did match")
55+
shouldNotMatchQueries := []Query{
56+
Query{
57+
Filters: []QueryFilter{
58+
QueryFilter{FieldName: "someStr", Value: "Pete", Operator: "="},
59+
},
60+
},
61+
Query{
62+
QueryString: "bla",
63+
Filters: []QueryFilter{
64+
QueryFilter{FieldName: "someStr", Value: "Pete", Operator: "="},
65+
},
66+
},
67+
Query{
68+
After: &nextHour,
69+
},
7170
}
72-
if MatchesQuery(lm, shouldNotMatchQuery2) {
73-
t.Errorf("Did match 2")
71+
for i, shouldMatch := range shouldMatchQueries {
72+
if !MatchesQuery(lm, shouldMatch) {
73+
t.Errorf("Did not match: %d: %+v", i, shouldMatch)
74+
}
7475
}
75-
if MatchesQuery(lm, shouldNotMatchQuery3) {
76-
t.Errorf("Did match 3")
76+
for i, shouldNotMatch := range shouldNotMatchQueries {
77+
if MatchesQuery(lm, shouldNotMatch) {
78+
t.Errorf("Did match: %d: %+v", i, shouldNotMatch)
79+
}
7780
}
7881
}
7982

‎pkg/backend/kibana/query.go‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit,
4242
if query.QueryString == "" {
4343
queryString = "*"
4444
}
45-
filterList := JsonList{
45+
mustFilters := JsonList{
4646
JsonObject{
4747
"query_string": JsonObject{
4848
"analyze_wildcard": true,
@@ -65,17 +65,29 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit,
6565
if query.Before != nil {
6666
rangeObj["range"].(JsonObject)["@timestamp"].(JsonObject)["lt"] = unixMillis(*query.Before)
6767
}
68-
filterList = append(filterList, rangeObj)
68+
mustFilters = append(mustFilters, rangeObj)
6969
}
70+
mustNotFilters := JsonList{}
7071
for _, filter := range query.Filters {
7172
m := JsonObject{}
72-
m[filter.FieldName] = JsonObject{
73-
"query": filter.Value,
74-
"type": "phrase",
73+
switch filter.Operator {
74+
case "=":
75+
m[filter.FieldName] = JsonObject{
76+
"query": filter.Value,
77+
"type": "phrase",
78+
}
79+
mustFilters = append(mustFilters, JsonObject{
80+
"match": m,
81+
})
82+
case "!=":
83+
m[filter.FieldName] = JsonObject{
84+
"query": filter.Value,
85+
"type": "phrase",
86+
}
87+
mustNotFilters = append(mustNotFilters, JsonObject{
88+
"match": m,
89+
})
7590
}
76-
filterList = append(filterList, JsonObject{
77-
"match": m,
78-
})
7991
}
8092
body, err := createMultiSearch(
8193
JsonObject{
@@ -94,7 +106,8 @@ func (client *Client) queryMessages(subIndex string, query common.Query) ([]Hit,
94106
},
95107
"query": JsonObject{
96108
"bool": JsonObject{
97-
"must": filterList,
109+
"must": mustFilters,
110+
"must_not": mustNotFilters,
98111
},
99112
},
100113
})
@@ -197,19 +210,20 @@ func (client *Client) querySubIndex(subIndex string, q common.Query) ([]common.L
197210

198211
allMessages := make([]common.LogMessage, 0, 200)
199212
for _, hit := range hits {
200-
var attributes map[string]interface{}
201-
var ts time.Time
202-
attributes = common.Project(hit.Source, q.SelectFields)
203-
ts, err = time.Parse(time.RFC3339, hit.Source["@timestamp"].(string))
213+
//var ts time.Time
214+
attributes := hit.Source
215+
ts, err := time.Parse(time.RFC3339, attributes["@timestamp"].(string))
204216
if err != nil {
205217
return nil, err
206218
}
207219
delete(attributes, "@timestamp")
208-
allMessages = append(allMessages, common.FlattenLogMessage(common.LogMessage{
220+
message := common.FlattenLogMessage(common.LogMessage{
209221
ID: hit.ID,
210222
Timestamp: ts,
211223
Attributes: attributes,
212-
}))
224+
})
225+
message.Attributes = common.Project(message.Attributes, q.SelectFields)
226+
allMessages = append(allMessages, message)
213227
}
214228
return allMessages, nil
215229
}

0 commit comments

Comments
 (0)