diff --git a/.gitignore b/.gitignore index 8899d84..253d47f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ go.work .direnv bin/ +tmp/ diff --git a/Makefile b/Makefile index 4b4e1b6..ba54943 100644 --- a/Makefile +++ b/Makefile @@ -8,15 +8,14 @@ export GOOSE_MIGRATION_DIR ?= schemas/migrations build: generate-go generate-web go build -o bin/bluemage ./go/cmd/bluemage/main.go -run: build - ARGS=$${ARGS:-serve} - ./bin/bluemage $$ARGS +run: + cd go && air run-web: generate-web cd web && npm run dev generate-go: migrate - rm -rf go/gen + rm -rf go/gen/* (cd ./schemas/proto && buf generate --template buf.gen.go.yaml .) (cd go/gen && bobgen-sqlite --config ../bobgen.yaml) (cd go && goverter gen -g 'output:file ../gen/converter/converter.go' -g 'output:package github.com/tigorlazuardi/bluemage/go/gen/converter' ./converter) diff --git a/README.md b/README.md index 9501f80..2c1b635 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,30 @@ # Bluemage -Reddit Image Downloader, but Codegen based \ No newline at end of file +Reddit Image Downloader, but Codegen based + +## Development Setup + +> Dependencies + +- Go 1.22.5+ +- [Goverter@v1.5.0] +- [Bob@v0.28.1] +- NodeJS 20+ +- ConnectRPC codegen related binaries: + - protoc-gen-go + - protoc-gen-connect-go + - protoc-gen-es + - protoc-gen-connect-es + +[Goverter@v1.5.0]: https://github.com/jmattheis/goverter/tree/v1.5.0 +[Bob@v0.28.1]: https://github.com/stephenafamo/bob/tree/v0.28.1 + +## Build + +Install dependencies, then run the following command: + +```sh +$ make build +``` + +If all success, the built binary will be available in the `bin` directory. diff --git a/flake.nix b/flake.nix index 2080eb5..adc48f8 100644 --- a/flake.nix +++ b/flake.nix @@ -71,6 +71,8 @@ impl goverter bobgen-sqlite + air + gopls ]; }; }; diff --git a/go.mod b/go.mod index bdb6a9e..fc58201 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,13 @@ require ( connectrpc.com/cors v0.1.0 github.com/aarondl/opt v0.0.0-20240623220848-083f18ab9536 github.com/bufbuild/protovalidate-go v0.6.3 + github.com/fatih/color v1.17.0 github.com/jaswdr/faker/v2 v2.3.0 github.com/mattn/go-sqlite3 v1.14.16 github.com/rs/cors v1.11.0 github.com/spf13/cobra v1.8.1 github.com/stephenafamo/bob v0.28.1 + github.com/tidwall/pretty v1.2.1 golang.org/x/net v0.23.0 google.golang.org/protobuf v1.34.2 ) @@ -22,11 +24,14 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/google/cel-go v0.20.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stephenafamo/scan v0.4.2 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect diff --git a/go.sum b/go.sum index c625d58..eaa52ca 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= @@ -56,6 +58,11 @@ github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA= github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8= github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -98,6 +105,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU= github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA= github.com/volatiletech/strmangle v0.0.6 h1:AdOYE3B2ygRDq4rXDij/MMwq6KVK/pWAYxpC7CLrkKQ= @@ -110,6 +119,8 @@ golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= diff --git a/go/.air.toml b/go/.air.toml new file mode 100644 index 0000000..307e6af --- /dev/null +++ b/go/.air.toml @@ -0,0 +1,51 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] +args_bin = ["serve"] +bin = "./tmp/main" +cmd = "go build -o ./tmp/main ./cmd/bluemage" +delay = 1000 +exclude_dir = ["assets", "tmp", "vendor", "testdata", "gen"] +exclude_file = [] +exclude_regex = ["_test.go"] +exclude_unchanged = false +follow_symlink = false +full_bin = "" +include_dir = [] +include_ext = ["go", "tpl", "tmpl", "html"] +include_file = [] +kill_delay = "0s" +log = "build-errors.log" +poll = false +poll_interval = 0 +post_cmd = [] +pre_cmd = ["cd .. && make generate-go"] +rerun = false +rerun_delay = 500 +send_interrupt = true +stop_on_error = true + +[color] +app = "" +build = "yellow" +main = "magenta" +runner = "green" +watcher = "cyan" + +[log] +main_only = false +time = false + +[misc] +clean_on_exit = false + +[proxy] +app_port = 0 +enabled = false +proxy_port = 0 + +[screen] +clear_on_rebuild = false +keep_scroll = true diff --git a/go/cmd/bluemage/serve/serve.go b/go/cmd/bluemage/serve/serve.go index c9d16f0..def6e72 100644 --- a/go/cmd/bluemage/serve/serve.go +++ b/go/cmd/bluemage/serve/serve.go @@ -5,6 +5,7 @@ import ( "errors" "log/slog" "net/http" + "os" "time" "connectrpc.com/connect" @@ -13,6 +14,7 @@ import ( "github.com/tigorlazuardi/bluemage/go/api" "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1/v1connect" "github.com/tigorlazuardi/bluemage/go/pkg/errs" + "github.com/tigorlazuardi/bluemage/go/pkg/log" "github.com/tigorlazuardi/bluemage/go/server" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" @@ -21,11 +23,15 @@ import ( var Cmd = &cobra.Command{ Use: "serve", RunE: func(cmd *cobra.Command, args []string) error { - db, err := bob.Open("sqlite3", "file:go/data.db") + db, err := bob.Open("sqlite3", "file:data.db") if err != nil { return errs.Wrap(err, "failed to open database") } + logOutput := log.WrapOsFile(os.Stderr) + prettyHandler := log.NewPrettyHandler(logOutput, nil) + slog.SetDefault(slog.New(prettyHandler)) + api := &api.API{ DB: db, } diff --git a/go/pkg/log/log.go b/go/pkg/log/log.go new file mode 100644 index 0000000..7330d54 --- /dev/null +++ b/go/pkg/log/log.go @@ -0,0 +1 @@ +package log diff --git a/go/pkg/log/pretty.go b/go/pkg/log/pretty.go new file mode 100644 index 0000000..e0ef96a --- /dev/null +++ b/go/pkg/log/pretty.go @@ -0,0 +1,226 @@ +package log + +import ( + "bytes" + "context" + "io" + "log/slog" + "os" + "strings" + "sync" + "sync/atomic" + + "github.com/fatih/color" + "github.com/tidwall/pretty" + "github.com/tigorlazuardi/bluemage/go/pkg/caller" +) + +type PrettyHandler struct { + opts *slog.HandlerOptions + output WriteSyncer + replaceAttr func(groups []string, attr slog.Attr) slog.Attr + withAttrs []slog.Attr + withGroup []string + isFlushing atomic.Bool + flushingChan chan struct{} + wg sync.WaitGroup +} + +// NewPrettyHandler creates a human friendly readable logs. +func NewPrettyHandler(writer WriteSyncer, opts *slog.HandlerOptions) *PrettyHandler { + if opts == nil { + opts = &slog.HandlerOptions{Level: slog.LevelDebug} + } + if opts.ReplaceAttr == nil { + opts.ReplaceAttr = func(groups []string, attr slog.Attr) slog.Attr { return attr } + } + return &PrettyHandler{ + opts: opts, + output: writer, + replaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + if len(groups) > 0 { + return opts.ReplaceAttr(groups, attr) + } + switch attr.Key { + case slog.TimeKey, slog.LevelKey, slog.SourceKey, slog.MessageKey: + return slog.Attr{} + default: + return opts.ReplaceAttr(groups, attr) + } + }, + flushingChan: make(chan struct{}), + } +} + +// Flush waits until all logs are written to the underlying writer +// and then flush the buffer. +// +// Flush blocks other log writes until it's done. +func (pr *PrettyHandler) Flush() error { + pr.isFlushing.Store(true) + defer func() { pr.isFlushing.Store(false) }() + pr.wg.Wait() + err := pr.output.Sync() + for { + select { + // signal to handlers that flushing is done. + case pr.flushingChan <- struct{}{}: + default: + return err + } + } +} + +// Sync attemps to flush the buffer of the underlying writer. +func (pr *PrettyHandler) Sync() error { + return pr.output.Sync() +} + +// Enabled implements slog.Handler interface. +func (pr *PrettyHandler) Enabled(ctx context.Context, lvl slog.Level) bool { + return pr.opts.Level.Level() <= lvl +} + +var bufferPool = sync.Pool{ + New: func() interface{} { + buf := &bytes.Buffer{} + buf.Grow(1024) + return buf + }, +} + +func putBuffer(buf *bytes.Buffer) { + const limit = 1024 * 512 // 512KB + if buf.Cap() < limit { + buf.Reset() + bufferPool.Put(buf) + } +} + +var jsonColorStyle = &pretty.Style{ + Key: [2]string{"\x1B[95m", "\x1B[0m"}, + String: [2]string{"\x1B[32m", "\x1B[0m"}, + Number: [2]string{"\x1B[33m", "\x1B[0m"}, + True: [2]string{"\x1B[36m", "\x1B[0m"}, + False: [2]string{"\x1B[36m", "\x1B[0m"}, + Null: [2]string{"\x1B[2m", "\x1B[0m"}, + Escape: [2]string{"\x1B[35m", "\x1B[0m"}, + Brackets: [2]string{"\x1B[0m", "\x1B[0m"}, + Append: pretty.TerminalStyle.Append, +} + +var jsonPrettyOpts = &pretty.Options{ + Width: 80, + Prefix: "", + Indent: " ", + SortKeys: false, +} + +// Handle implements slog.Handler interface. +func (pr *PrettyHandler) Handle(ctx context.Context, record slog.Record) error { + if pr.isFlushing.Load() { + // pr is currently flushing. Wait until the flushing process is done. + <-pr.flushingChan + } + pr.wg.Add(1) + defer pr.wg.Done() + + var levelColor *color.Color + switch { + case record.Level >= slog.LevelError: + levelColor = color.New(color.FgRed) + case record.Level >= slog.LevelWarn: + levelColor = color.New(color.FgYellow) + case record.Level >= slog.LevelInfo: + levelColor = color.New(color.FgGreen) + default: + levelColor = color.New(color.FgWhite) + } + + buf := bufferPool.Get().(*bytes.Buffer) + jsonBuf := bufferPool.Get().(*bytes.Buffer) + defer putBuffer(buf) + defer putBuffer(jsonBuf) + + if record.PC != 0 && pr.opts.AddSource { + frame := caller.From(record.PC).Frame + levelColor.Fprint(buf, frame.File) + levelColor.Fprint(buf, ":") + levelColor.Fprint(buf, frame.Line) + levelColor.Fprint(buf, " -- ") + split := strings.Split(frame.Function, string(os.PathSeparator)) + fnName := split[len(split)-1] + levelColor.Fprint(buf, fnName) + buf.WriteByte('\n') + } + + if !record.Time.IsZero() { + buf.WriteString(record.Time.Format("[2006-01-02 15:04:05] ")) + } + + buf.WriteByte('[') + levelColor.Add(color.Bold).Fprint(buf, record.Level.String()) + buf.WriteString("] ") + + if record.Message != "" { + buf.WriteString(record.Message) + } + + buf.WriteByte('\n') + + serializer := pr.createSerializer(jsonBuf) + _ = serializer.Handle(ctx, record) + if jsonBuf.Len() > 3 { // Ignore empty json like "{}\n" + jsonData := jsonBuf.Bytes() + jsonData = pretty.PrettyOptions(jsonData, jsonPrettyOpts) + jsonData = pretty.Color(jsonData, jsonColorStyle) + buf.Write(jsonData) + } + buf.WriteByte('\n') + + _, err := buf.WriteTo(pr.output) + return err +} + +func (pr *PrettyHandler) createSerializer(w io.Writer) slog.Handler { + var jsonHandler slog.Handler = slog.NewJSONHandler(w, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: pr.replaceAttr, + }) + + if len(pr.withAttrs) > 0 { + jsonHandler = jsonHandler.WithAttrs(pr.withAttrs) + } + + if len(pr.withGroup) > 0 { + for _, group := range pr.withGroup { + jsonHandler = jsonHandler.WithGroup(group) + } + } + + return jsonHandler +} + +func (pr *PrettyHandler) clone() *PrettyHandler { + return &PrettyHandler{ + opts: pr.opts, + output: pr.output, + replaceAttr: pr.replaceAttr, + withAttrs: pr.withAttrs, + withGroup: pr.withGroup, + } +} + +// WithAttrs implements slog.Handler interface. +func (pr *PrettyHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + p := pr.clone() + p.withAttrs = append(p.withAttrs, attrs...) + return p +} + +// WithGroup implements slog.Handler interface. +func (pr *PrettyHandler) WithGroup(name string) slog.Handler { + p := pr.clone() + p.withGroup = append(p.withGroup, name) + return p +} diff --git a/go/pkg/log/writer.go b/go/pkg/log/writer.go new file mode 100644 index 0000000..f21bd60 --- /dev/null +++ b/go/pkg/log/writer.go @@ -0,0 +1,69 @@ +package log + +import ( + "io" + "os" + "sync" +) + +type WriteSyncer interface { + io.Writer + Sync() error +} + +// WrapOsFile wraps an *os.File in a WriteSyncer. +// +// To support multithreaded logging and to support +// flushing the buffer before the program exits. +func WrapOsFile(f *os.File) WriteSyncer { + return Lock(f) +} + +// AddSync converts an io.Writer to a WriteSyncer. It attempts to be +// intelligent: if the concrete type of the io.Writer implements WriteSyncer, +// we'll use the existing Sync method. If it doesn't, we'll add a no-op Sync. +func AddSync(w io.Writer) WriteSyncer { + switch w := w.(type) { + case WriteSyncer: + return w + default: + return writerWrapper{w} + } +} + +type lockedWriteSyncer struct { + sync.Mutex + ws WriteSyncer +} + +// Lock wraps a WriteSyncer in a mutex to make it safe for concurrent use. In +// particular, *os.Files must be locked before use. +func Lock(ws WriteSyncer) WriteSyncer { + if _, ok := ws.(*lockedWriteSyncer); ok { + // no need to layer on another lock + return ws + } + return &lockedWriteSyncer{ws: ws} +} + +func (s *lockedWriteSyncer) Write(bs []byte) (int, error) { + s.Lock() + n, err := s.ws.Write(bs) + s.Unlock() + return n, err +} + +func (s *lockedWriteSyncer) Sync() error { + s.Lock() + err := s.ws.Sync() + s.Unlock() + return err +} + +type writerWrapper struct { + io.Writer +} + +func (w writerWrapper) Sync() error { + return nil +} diff --git a/go/server/interceptor.go b/go/server/interceptor.go index 193848a..4c738fb 100644 --- a/go/server/interceptor.go +++ b/go/server/interceptor.go @@ -16,19 +16,19 @@ func LogInterceptor() connect.UnaryInterceptorFunc { start := time.Now() resp, err := next(ctx, ar) dur := time.Since(start) - millisecondsFloat := float64(dur) / float64(time.Millisecond) + durFloat := float64(dur) / float64(time.Second) if err != nil { slog.Error("RPC Error", "procedure", ar.Spec().Procedure, "method", ar.HTTPMethod(), - "duration", fmt.Sprintf("%.3fs", millisecondsFloat), + "duration", fmt.Sprintf("%.3fs", durFloat), "error", errs.DrillToError(err), ) } else { slog.Info("RPC Call", "procedure", ar.Spec().Procedure, "method", ar.HTTPMethod(), - "duration", fmt.Sprintf("%.3fs", millisecondsFloat), + "duration", fmt.Sprintf("%.3fs", durFloat), ) }