go: added air and pretty logging

This commit is contained in:
Tigor Hutasuhut 2024-08-07 10:41:00 +07:00
parent 261361e5a7
commit f559ad7ca0
12 changed files with 407 additions and 9 deletions

1
.gitignore vendored
View file

@ -23,3 +23,4 @@ go.work
.direnv
bin/
tmp/

View file

@ -8,15 +8,14 @@ export GOOSE_MIGRATION_DIR ?= schemas/migrations
build: generate-go generate-web
go build -o bin/bluemage ./go/cmd/bluemage/main.go
run: build
ARGS=$${ARGS:-serve}
./bin/bluemage $$ARGS
run:
cd go && air
run-web: generate-web
cd web && npm run dev
generate-go: migrate
rm -rf go/gen
rm -rf go/gen/*
(cd ./schemas/proto && buf generate --template buf.gen.go.yaml .)
(cd go/gen && bobgen-sqlite --config ../bobgen.yaml)
(cd go && goverter gen -g 'output:file ../gen/converter/converter.go' -g 'output:package github.com/tigorlazuardi/bluemage/go/gen/converter' ./converter)

View file

@ -1,3 +1,30 @@
# Bluemage
Reddit Image Downloader, but Codegen based
Reddit Image Downloader, but Codegen based
## Development Setup
> Dependencies
- Go 1.22.5+
- [Goverter@v1.5.0]
- [Bob@v0.28.1]
- NodeJS 20+
- ConnectRPC codegen related binaries:
- protoc-gen-go
- protoc-gen-connect-go
- protoc-gen-es
- protoc-gen-connect-es
[Goverter@v1.5.0]: https://github.com/jmattheis/goverter/tree/v1.5.0
[Bob@v0.28.1]: https://github.com/stephenafamo/bob/tree/v0.28.1
## Build
Install dependencies, then run the following command:
```sh
$ make build
```
If all success, the built binary will be available in the `bin` directory.

View file

@ -71,6 +71,8 @@
impl
goverter
bobgen-sqlite
air
gopls
];
};
};

5
go.mod
View file

@ -8,11 +8,13 @@ require (
connectrpc.com/cors v0.1.0
github.com/aarondl/opt v0.0.0-20240623220848-083f18ab9536
github.com/bufbuild/protovalidate-go v0.6.3
github.com/fatih/color v1.17.0
github.com/jaswdr/faker/v2 v2.3.0
github.com/mattn/go-sqlite3 v1.14.16
github.com/rs/cors v1.11.0
github.com/spf13/cobra v1.8.1
github.com/stephenafamo/bob v0.28.1
github.com/tidwall/pretty v1.2.1
golang.org/x/net v0.23.0
google.golang.org/protobuf v1.34.2
)
@ -22,11 +24,14 @@ require (
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/google/cel-go v0.20.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stephenafamo/scan v0.4.2 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect

11
go.sum
View file

@ -26,6 +26,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c=
@ -56,6 +58,11 @@ github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf
github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8=
github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
@ -98,6 +105,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU=
github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA=
github.com/volatiletech/strmangle v0.0.6 h1:AdOYE3B2ygRDq4rXDij/MMwq6KVK/pWAYxpC7CLrkKQ=
@ -110,6 +119,8 @@ golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

51
go/.air.toml Normal file
View file

@ -0,0 +1,51 @@
root = "."
testdata_dir = "testdata"
tmp_dir = "tmp"
[build]
args_bin = ["serve"]
bin = "./tmp/main"
cmd = "go build -o ./tmp/main ./cmd/bluemage"
delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata", "gen"]
exclude_file = []
exclude_regex = ["_test.go"]
exclude_unchanged = false
follow_symlink = false
full_bin = ""
include_dir = []
include_ext = ["go", "tpl", "tmpl", "html"]
include_file = []
kill_delay = "0s"
log = "build-errors.log"
poll = false
poll_interval = 0
post_cmd = []
pre_cmd = ["cd .. && make generate-go"]
rerun = false
rerun_delay = 500
send_interrupt = true
stop_on_error = true
[color]
app = ""
build = "yellow"
main = "magenta"
runner = "green"
watcher = "cyan"
[log]
main_only = false
time = false
[misc]
clean_on_exit = false
[proxy]
app_port = 0
enabled = false
proxy_port = 0
[screen]
clear_on_rebuild = false
keep_scroll = true

View file

@ -5,6 +5,7 @@ import (
"errors"
"log/slog"
"net/http"
"os"
"time"
"connectrpc.com/connect"
@ -13,6 +14,7 @@ import (
"github.com/tigorlazuardi/bluemage/go/api"
"github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1/v1connect"
"github.com/tigorlazuardi/bluemage/go/pkg/errs"
"github.com/tigorlazuardi/bluemage/go/pkg/log"
"github.com/tigorlazuardi/bluemage/go/server"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
@ -21,11 +23,15 @@ import (
var Cmd = &cobra.Command{
Use: "serve",
RunE: func(cmd *cobra.Command, args []string) error {
db, err := bob.Open("sqlite3", "file:go/data.db")
db, err := bob.Open("sqlite3", "file:data.db")
if err != nil {
return errs.Wrap(err, "failed to open database")
}
logOutput := log.WrapOsFile(os.Stderr)
prettyHandler := log.NewPrettyHandler(logOutput, nil)
slog.SetDefault(slog.New(prettyHandler))
api := &api.API{
DB: db,
}

1
go/pkg/log/log.go Normal file
View file

@ -0,0 +1 @@
package log

226
go/pkg/log/pretty.go Normal file
View file

@ -0,0 +1,226 @@
package log
import (
"bytes"
"context"
"io"
"log/slog"
"os"
"strings"
"sync"
"sync/atomic"
"github.com/fatih/color"
"github.com/tidwall/pretty"
"github.com/tigorlazuardi/bluemage/go/pkg/caller"
)
type PrettyHandler struct {
opts *slog.HandlerOptions
output WriteSyncer
replaceAttr func(groups []string, attr slog.Attr) slog.Attr
withAttrs []slog.Attr
withGroup []string
isFlushing atomic.Bool
flushingChan chan struct{}
wg sync.WaitGroup
}
// NewPrettyHandler creates a human friendly readable logs.
func NewPrettyHandler(writer WriteSyncer, opts *slog.HandlerOptions) *PrettyHandler {
if opts == nil {
opts = &slog.HandlerOptions{Level: slog.LevelDebug}
}
if opts.ReplaceAttr == nil {
opts.ReplaceAttr = func(groups []string, attr slog.Attr) slog.Attr { return attr }
}
return &PrettyHandler{
opts: opts,
output: writer,
replaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if len(groups) > 0 {
return opts.ReplaceAttr(groups, attr)
}
switch attr.Key {
case slog.TimeKey, slog.LevelKey, slog.SourceKey, slog.MessageKey:
return slog.Attr{}
default:
return opts.ReplaceAttr(groups, attr)
}
},
flushingChan: make(chan struct{}),
}
}
// Flush waits until all logs are written to the underlying writer
// and then flush the buffer.
//
// Flush blocks other log writes until it's done.
func (pr *PrettyHandler) Flush() error {
pr.isFlushing.Store(true)
defer func() { pr.isFlushing.Store(false) }()
pr.wg.Wait()
err := pr.output.Sync()
for {
select {
// signal to handlers that flushing is done.
case pr.flushingChan <- struct{}{}:
default:
return err
}
}
}
// Sync attemps to flush the buffer of the underlying writer.
func (pr *PrettyHandler) Sync() error {
return pr.output.Sync()
}
// Enabled implements slog.Handler interface.
func (pr *PrettyHandler) Enabled(ctx context.Context, lvl slog.Level) bool {
return pr.opts.Level.Level() <= lvl
}
var bufferPool = sync.Pool{
New: func() interface{} {
buf := &bytes.Buffer{}
buf.Grow(1024)
return buf
},
}
func putBuffer(buf *bytes.Buffer) {
const limit = 1024 * 512 // 512KB
if buf.Cap() < limit {
buf.Reset()
bufferPool.Put(buf)
}
}
var jsonColorStyle = &pretty.Style{
Key: [2]string{"\x1B[95m", "\x1B[0m"},
String: [2]string{"\x1B[32m", "\x1B[0m"},
Number: [2]string{"\x1B[33m", "\x1B[0m"},
True: [2]string{"\x1B[36m", "\x1B[0m"},
False: [2]string{"\x1B[36m", "\x1B[0m"},
Null: [2]string{"\x1B[2m", "\x1B[0m"},
Escape: [2]string{"\x1B[35m", "\x1B[0m"},
Brackets: [2]string{"\x1B[0m", "\x1B[0m"},
Append: pretty.TerminalStyle.Append,
}
var jsonPrettyOpts = &pretty.Options{
Width: 80,
Prefix: "",
Indent: " ",
SortKeys: false,
}
// Handle implements slog.Handler interface.
func (pr *PrettyHandler) Handle(ctx context.Context, record slog.Record) error {
if pr.isFlushing.Load() {
// pr is currently flushing. Wait until the flushing process is done.
<-pr.flushingChan
}
pr.wg.Add(1)
defer pr.wg.Done()
var levelColor *color.Color
switch {
case record.Level >= slog.LevelError:
levelColor = color.New(color.FgRed)
case record.Level >= slog.LevelWarn:
levelColor = color.New(color.FgYellow)
case record.Level >= slog.LevelInfo:
levelColor = color.New(color.FgGreen)
default:
levelColor = color.New(color.FgWhite)
}
buf := bufferPool.Get().(*bytes.Buffer)
jsonBuf := bufferPool.Get().(*bytes.Buffer)
defer putBuffer(buf)
defer putBuffer(jsonBuf)
if record.PC != 0 && pr.opts.AddSource {
frame := caller.From(record.PC).Frame
levelColor.Fprint(buf, frame.File)
levelColor.Fprint(buf, ":")
levelColor.Fprint(buf, frame.Line)
levelColor.Fprint(buf, " -- ")
split := strings.Split(frame.Function, string(os.PathSeparator))
fnName := split[len(split)-1]
levelColor.Fprint(buf, fnName)
buf.WriteByte('\n')
}
if !record.Time.IsZero() {
buf.WriteString(record.Time.Format("[2006-01-02 15:04:05] "))
}
buf.WriteByte('[')
levelColor.Add(color.Bold).Fprint(buf, record.Level.String())
buf.WriteString("] ")
if record.Message != "" {
buf.WriteString(record.Message)
}
buf.WriteByte('\n')
serializer := pr.createSerializer(jsonBuf)
_ = serializer.Handle(ctx, record)
if jsonBuf.Len() > 3 { // Ignore empty json like "{}\n"
jsonData := jsonBuf.Bytes()
jsonData = pretty.PrettyOptions(jsonData, jsonPrettyOpts)
jsonData = pretty.Color(jsonData, jsonColorStyle)
buf.Write(jsonData)
}
buf.WriteByte('\n')
_, err := buf.WriteTo(pr.output)
return err
}
func (pr *PrettyHandler) createSerializer(w io.Writer) slog.Handler {
var jsonHandler slog.Handler = slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: slog.LevelDebug,
ReplaceAttr: pr.replaceAttr,
})
if len(pr.withAttrs) > 0 {
jsonHandler = jsonHandler.WithAttrs(pr.withAttrs)
}
if len(pr.withGroup) > 0 {
for _, group := range pr.withGroup {
jsonHandler = jsonHandler.WithGroup(group)
}
}
return jsonHandler
}
func (pr *PrettyHandler) clone() *PrettyHandler {
return &PrettyHandler{
opts: pr.opts,
output: pr.output,
replaceAttr: pr.replaceAttr,
withAttrs: pr.withAttrs,
withGroup: pr.withGroup,
}
}
// WithAttrs implements slog.Handler interface.
func (pr *PrettyHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
p := pr.clone()
p.withAttrs = append(p.withAttrs, attrs...)
return p
}
// WithGroup implements slog.Handler interface.
func (pr *PrettyHandler) WithGroup(name string) slog.Handler {
p := pr.clone()
p.withGroup = append(p.withGroup, name)
return p
}

69
go/pkg/log/writer.go Normal file
View file

@ -0,0 +1,69 @@
package log
import (
"io"
"os"
"sync"
)
type WriteSyncer interface {
io.Writer
Sync() error
}
// WrapOsFile wraps an *os.File in a WriteSyncer.
//
// To support multithreaded logging and to support
// flushing the buffer before the program exits.
func WrapOsFile(f *os.File) WriteSyncer {
return Lock(f)
}
// AddSync converts an io.Writer to a WriteSyncer. It attempts to be
// intelligent: if the concrete type of the io.Writer implements WriteSyncer,
// we'll use the existing Sync method. If it doesn't, we'll add a no-op Sync.
func AddSync(w io.Writer) WriteSyncer {
switch w := w.(type) {
case WriteSyncer:
return w
default:
return writerWrapper{w}
}
}
type lockedWriteSyncer struct {
sync.Mutex
ws WriteSyncer
}
// Lock wraps a WriteSyncer in a mutex to make it safe for concurrent use. In
// particular, *os.Files must be locked before use.
func Lock(ws WriteSyncer) WriteSyncer {
if _, ok := ws.(*lockedWriteSyncer); ok {
// no need to layer on another lock
return ws
}
return &lockedWriteSyncer{ws: ws}
}
func (s *lockedWriteSyncer) Write(bs []byte) (int, error) {
s.Lock()
n, err := s.ws.Write(bs)
s.Unlock()
return n, err
}
func (s *lockedWriteSyncer) Sync() error {
s.Lock()
err := s.ws.Sync()
s.Unlock()
return err
}
type writerWrapper struct {
io.Writer
}
func (w writerWrapper) Sync() error {
return nil
}

View file

@ -16,19 +16,19 @@ func LogInterceptor() connect.UnaryInterceptorFunc {
start := time.Now()
resp, err := next(ctx, ar)
dur := time.Since(start)
millisecondsFloat := float64(dur) / float64(time.Millisecond)
durFloat := float64(dur) / float64(time.Second)
if err != nil {
slog.Error("RPC Error",
"procedure", ar.Spec().Procedure,
"method", ar.HTTPMethod(),
"duration", fmt.Sprintf("%.3fs", millisecondsFloat),
"duration", fmt.Sprintf("%.3fs", durFloat),
"error", errs.DrillToError(err),
)
} else {
slog.Info("RPC Call",
"procedure", ar.Spec().Procedure,
"method", ar.HTTPMethod(),
"duration", fmt.Sprintf("%.3fs", millisecondsFloat),
"duration", fmt.Sprintf("%.3fs", durFloat),
)
}