Compare commits

...

2 commits

15 changed files with 160 additions and 112 deletions

1
go.mod
View file

@ -20,6 +20,7 @@ require (
)
require (
connectrpc.com/validate v0.1.0 // indirect
github.com/aarondl/json v0.0.0-20221020222930-8b0db17ef1bf // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/google/cel-go v0.20.1 // indirect

3
go.sum
View file

@ -4,6 +4,8 @@ connectrpc.com/connect v1.16.2 h1:ybd6y+ls7GOlb7Bh5C8+ghA6SvCBajHwxssO2CGFjqE=
connectrpc.com/connect v1.16.2/go.mod h1:n2kgwskMHXC+lVqb18wngEpF95ldBHXjZYJussz5FRc=
connectrpc.com/cors v0.1.0 h1:f3gTXJyDZPrDIZCQ567jxfD9PAIpopHiRDnJRt3QuOQ=
connectrpc.com/cors v0.1.0/go.mod h1:v8SJZCPfHtGH1zsm+Ttajpozd4cYIUryl4dFB6QEpfg=
connectrpc.com/validate v0.1.0 h1:r55jirxMK7HO/xZwVHj3w2XkVFarsUM77ZDy367NtH4=
connectrpc.com/validate v0.1.0/go.mod h1:GU47c9/x/gd+u9wRSPkrQOP46gx2rMN+Wo37EHgI3Ow=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@ -16,6 +18,7 @@ github.com/aarondl/opt v0.0.0-20240623220848-083f18ab9536 h1:vhpjulzH5Tr4S3uJ3Y/
github.com/aarondl/opt v0.0.0-20240623220848-083f18ab9536/go.mod h1:l4/5NZtYd/SIohsFhaJQQe+sPOTG22furpZ5FvcYOzk=
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM=
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/bufbuild/protovalidate-go v0.6.3 h1:wxQyzW035zM16Binbaz/nWAzS12dRIXhZdSUWRY7Fv0=

39
go/api/devices_count.go Normal file
View file

@ -0,0 +1,39 @@
package api
import (
"context"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/sqlite"
"github.com/stephenafamo/bob/dialect/sqlite/dialect"
"github.com/stephenafamo/bob/dialect/sqlite/sm"
"github.com/tigorlazuardi/bluemage/go/gen/models"
device "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1"
"github.com/tigorlazuardi/bluemage/go/pkg/errs"
)
func queryFromCountDevicesRequest(req *device.CountDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) {
switch req.Disabled {
case device.DisabledFilter_DISABLED_FILTER_TRUE:
expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1))
case device.DisabledFilter_DISABLED_FILTER_FALSE:
expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0))
}
if req.Search != "" {
arg := sqlite.Arg("%" + req.Search + "%")
expr = append(expr,
sm.Where(
models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)),
),
)
}
return expr
}
func (api *API) DevicesCount(ctx context.Context, request *device.CountDevicesRequest) (uint64, error) {
count, err := models.Devices.Query(ctx, api.DB, queryFromCountDevicesRequest(request)...).Count()
if err != nil {
return 0, errs.Wrapw(err, "failed to count devices", "request", request)
}
return uint64(count), nil
}

View file

@ -14,7 +14,21 @@ import (
)
func queryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) {
expr = countQueryFromListDeviceRequest(req)
switch req.Disabled {
case device.DisabledFilter_DISABLED_FILTER_TRUE:
expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1))
case device.DisabledFilter_DISABLED_FILTER_FALSE:
expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0))
}
if req.Search != "" {
arg := sqlite.Arg("%" + req.Search + "%")
expr = append(expr,
sm.Where(
models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)),
),
)
}
if req.Limit > 0 {
expr = append(expr, sm.Limit(req.Limit))
@ -41,46 +55,12 @@ func queryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[
return expr
}
func countQueryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) {
switch req.Disabled {
case device.DisabledFilter_DISABLED_FILTER_TRUE:
expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1))
case device.DisabledFilter_DISABLED_FILTER_FALSE:
expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0))
}
if req.Search != "" {
arg := sqlite.Arg("%" + req.Search + "%")
expr = append(expr,
sm.Where(
models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)),
),
)
}
return expr
}
func (api *API) DevicesList(ctx context.Context, req *device.ListDevicesRequest) (resp *device.ListDevicesResponse, err error) {
resp = &device.ListDevicesResponse{}
results, err := models.Devices.Query(ctx, api.DB, queryFromListDeviceRequest(req)...).All()
if err != nil {
return resp, errs.Wrapw(err, "failed to list devices", "request", req)
}
if req.Disabled == device.DisabledFilter_DISABLED_FILTER_UNSPECIFIED && req.Search == "" {
const metricName = "devices.count"
metric, err := models.FindMetric(ctx, api.DB, metricName)
if err != nil {
return resp, errs.Wrapw(err, "failed to find devices count metric", "metric", metricName)
}
resp.Count = uint64(metric.Value)
} else {
count, err := models.Devices.Query(ctx, api.DB, countQueryFromListDeviceRequest(req)...).Count()
if err != nil {
return resp, errs.Wrapw(err, "failed to count query result")
}
resp.Count = uint64(count)
}
for _, result := range results {
resp.Devices = append(resp.Devices, convert.ModelsDeviceToGetDeviceResponse(result))
}

View file

@ -10,6 +10,7 @@ import (
"time"
"connectrpc.com/connect"
"connectrpc.com/validate"
sqldblogger "github.com/simukti/sqldb-logger"
"github.com/spf13/cobra"
"github.com/stephenafamo/bob"
@ -51,8 +52,16 @@ var Cmd = &cobra.Command{
},
}
validationInterceptor, err := validate.NewInterceptor()
if err != nil {
return errs.Wrap(err, "failed to create validation interceptor")
}
mux := http.NewServeMux()
mux.Handle(v1connect.NewDeviceServiceHandler(handler, connect.WithInterceptors(server.LogInterceptor())))
mux.Handle(v1connect.NewDeviceServiceHandler(handler, connect.WithInterceptors(
validationInterceptor,
server.LogInterceptor(),
)))
server := &http.Server{
Addr: ":8080",
@ -72,7 +81,8 @@ var Cmd = &cobra.Command{
if err != nil && !errors.Is(err, http.ErrServerClosed) {
return errs.Wrap(err, "failed to serve")
}
return errors.Join(sqldb.Close())
slog.Info("ConnectRPC server stopped")
return errors.Join(sqldb.Close(), prettyHandler.Flush())
},
SilenceUsage: true,
}

View file

@ -14,7 +14,7 @@ import (
// goverter:extend PtrFloat64ToOmitFloat64
// goverter:extend PtrInt32ToOmitInt32
type DeviceConverter interface {
// goverter:ignore CreatedAt UpdatedAt
// goverter:ignore CreatedAt UpdatedAt R
// goverter:map Nsfw NSFW
CreateDeviceRequestToModelsDevice(*device.CreateDeviceRequest) *models.Device
// goverter:ignore state sizeCache unknownFields

View file

@ -4,7 +4,6 @@ import (
"context"
"connectrpc.com/connect"
"github.com/bufbuild/protovalidate-go"
"github.com/tigorlazuardi/bluemage/go/api"
"github.com/tigorlazuardi/bluemage/go/gen/converter"
device "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1"
@ -20,19 +19,12 @@ type DeviceHandler struct {
v1connect.UnimplementedDeviceServiceHandler
}
var (
convert converter.DeviceConverterImpl
validate, _ = protovalidate.New()
)
var convert converter.DeviceConverterImpl
// CreateDevice implements v1connect.DeviceServiceHandler.
func (d *DeviceHandler) CreateDevice(ctx context.Context, request *connect.Request[device.CreateDeviceRequest]) (*connect.Response[device.CreateDeviceResponse], error) {
err := validate.Validate(request.Msg)
if err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
dev := convert.CreateDeviceRequestToModelsDevice(request.Msg)
dev, err = d.API.DevicesCreate(ctx, dev)
dev, err := d.API.DevicesCreate(ctx, dev)
if err != nil {
return nil, errs.IntoConnectError(err)
}
@ -42,11 +34,6 @@ func (d *DeviceHandler) CreateDevice(ctx context.Context, request *connect.Reque
// GetDevice implements v1connect.DeviceServiceHandler.
func (d *DeviceHandler) GetDevice(ctx context.Context, request *connect.Request[device.GetDeviceRequest]) (*connect.Response[device.GetDeviceResponse], error) {
err := validate.Validate(request.Msg)
if err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
dev, err := d.API.GetDevice(ctx, request.Msg.Slug)
if err != nil {
return nil, errs.IntoConnectError(err)
@ -86,10 +73,6 @@ func (de *DeviceHandler) UpdateDevice(ctx context.Context, request *connect.Requ
func (de *DeviceHandler) DeviceExists(ctx context.Context, request *connect.Request[device.DeviceExistsRequest]) (*connect.Response[device.DeviceExistsResponse], error) {
slug := request.Msg.Slug
if err := validate.Validate(request.Msg); err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, err)
}
exists, err := de.API.DevicesExist(ctx, slug)
if err != nil {
return nil, errs.IntoConnectError(err)
@ -100,3 +83,16 @@ func (de *DeviceHandler) DeviceExists(ctx context.Context, request *connect.Requ
}
return connect.NewResponse(resp), nil
}
// CountDevices count the number of devices.
func (de *DeviceHandler) CountDevices(ctx context.Context, request *connect.Request[device.CountDevicesRequest]) (*connect.Response[device.CountDevicesResponse], error) {
count, err := de.API.DevicesCount(ctx, request.Msg)
if err != nil {
return nil, errs.IntoConnectError(err)
}
resp := &device.CountDevicesResponse{
Count: count,
}
return connect.NewResponse(resp), nil
}

View file

@ -1,21 +0,0 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE metrics (
name VARCHAR(255) NOT NULL PRIMARY KEY COLLATE NOCASE,
value BIGINT DEFAULT 0 NOT NULL,
created_at BIGINT DEFAULT (strftime('%s', 'now')) NOT NULL,
updated_at BIGINT DEFAULT (strftime('%s', 'now')) NOT NULL
);
CREATE UNIQUE INDEX idx_metrics_name ON metrics(name);
CREATE TRIGGER update_metrics_timestamp_after_update AFTER UPDATE ON metrics FOR EACH ROW
BEGIN
UPDATE metrics SET updated_at = (strftime('%s', 'now')) WHERE name = old.name;
END;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE metrics;
-- +goose StatementEnd

View file

@ -12,22 +12,6 @@ CREATE TABLE subreddits (
);
CREATE UNIQUE INDEX idx_subreddits_name ON subreddits(name);
CREATE TRIGGER subreddits_update_timestamp AFTER UPDATE ON subreddits FOR EACH ROW
BEGIN
UPDATE subreddits SET updated_at = (strftime('%s', 'now')) WHERE name = old.name;
END;
CREATE TRIGGER subreddits_insert_create_total_image_metrics AFTER INSERT on subreddits FOR EACH ROW
BEGIN
INSERT INTO metrics (name, value)
VALUES (CONCAT('subreddits.', NEW.name, '.total_images'), 0);
END;
CREATE TRIGGER subreddits_delete_total_image_metrics AFTER DELETE on subreddits FOR EACH ROW
BEGIN
DELETE FROM metrics WHERE name = CONCAT('subreddits.', OLD.name, '.total_images');
END;
-- +goose StatementEnd
-- +goose Down

View file

@ -17,8 +17,6 @@ CREATE TABLE devices (
updated_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now'))
);
INSERT INTO metrics(name) VALUES ('devices.count');
CREATE UNIQUE INDEX idx_devices_unique_slug ON devices(slug);
CREATE TRIGGER devices_update_timestamp AFTER UPDATE ON devices FOR EACH ROW
@ -26,22 +24,9 @@ BEGIN
UPDATE devices SET updated_at = strftime('%s', 'now') WHERE slug = old.slug;
END;
CREATE TRIGGER devices_insert_create_total_image_metrics AFTER INSERT on devices FOR EACH ROW
BEGIN
INSERT INTO metrics (name, value)
VALUES (CONCAT('devices.', NEW.slug, '.total_images'), 0);
UPDATE metrics SET value = value + 1 WHERE name = 'devices.count';
END;
CREATE TRIGGER devices_delete_total_image_metrics AFTER DELETE on devices FOR EACH ROW
BEGIN
DELETE FROM metrics WHERE name = CONCAT('devices.', OLD.slug, '.total_images');
UPDATE metrics SET value = value - 1 WHERE name = 'devices.count';
END;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE devices;
DELETE FROM metrics WHERE name = 'devices.count';
-- +goose StatementEnd

View file

@ -0,0 +1,42 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE images(
id INTEGER PRIMARY KEY,
subreddit VARCHAR(255) NOT NULL COLLATE NOCASE,
device VARCHAR(250) NOT NULL COLLATE NOCASE,
post_title VARCHAR(255) NOT NULL,
post_name VARCHAR(255) NOT NULL,
post_url VARCHAR(255) NOT NULL,
post_created BIGINT NOT NULL DEFAULT CURRENT_TIMESTAMP,
post_author VARCHAR(50) NOT NULL,
post_author_url VARCHAR(255) NOT NULL,
image_relative_path VARCHAR(255) NOT NULL,
image_original_url VARCHAR(255) NOT NULL,
image_height INTEGER NOT NULL DEFAULT 0,
image_width INTEGER NOT NULL DEFAULT 0,
image_size BIGINT NOT NULL DEFAULT 0,
thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '',
nsfw INTEGER NOT NULL DEFAULT 0,
created_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
updated_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
CONSTRAINT fk_image_subreddit
FOREIGN KEY (subreddit)
REFERENCES subreddits(name)
ON DELETE CASCADE,
CONSTRAINT fk_image_devices_slug
FOREIGN KEY (device)
REFERENCES devices(slug)
ON DELETE CASCADE
);
CREATE INDEX idx_subreddit_images ON images(subreddit);
CREATE INDEX idx_subreddit_device_images ON images(device, subreddit);
CREATE INDEX idx_nsfw_images ON images(nsfw);
CREATE INDEX idx_images_created_at_nsfw ON images(created_at DESC, nsfw);
CREATE UNIQUE INDEX idx_unique_images_per_device ON images(device, post_name);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE images;
-- +goose StatementEnd

View file

@ -0,0 +1,26 @@
syntax = "proto3";
package device.v1;
import "device/v1/list.proto";
option go_package = "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1";
message CountDevicesRequest {
// Limits the counts to devices that have the given name.
// case insensitive.
//
// Ignored if empty.
//
// default: empty string.
string search = 1;
// disabled limit the counting to devices with the given status.
//
// If unspecified, devices with either status will be counted.
DisabledFilter disabled = 2;
}
message CountDevicesResponse {
uint64 count = 1;
}

View file

@ -2,6 +2,7 @@ syntax = "proto3";
package device.v1;
import "device/v1/count.proto";
import "device/v1/create.proto";
import "device/v1/exists.proto";
import "device/v1/get.proto";
@ -33,4 +34,7 @@ service DeviceService {
// DeviceExists checks if a device exists in the database.
rpc DeviceExists(DeviceExistsRequest) returns (DeviceExistsResponse) {}
// CountDevices count the number of devices.
rpc CountDevices(CountDevicesRequest) returns (CountDevicesResponse) {}
}

View file

@ -47,8 +47,7 @@ message ListDevicesRequest {
}
message ListDevicesResponse {
uint64 count = 1;
repeated GetDeviceResponse devices = 2;
repeated GetDeviceResponse devices = 1;
}
enum DisabledFilter {

View file

@ -7,7 +7,7 @@ import "buf/validate/validate.proto";
option go_package = "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1";
message UpdateDeviceRequest {
string slug = 1;
string slug = 1 [(buf.validate.field).string.min_len = 1];
DeviceSetter set = 2;
}