devices: added count devices endpoint

go.mod

@@ -20,6 +20,7 @@ require (
 )
 
 require (
+	connectrpc.com/validate v0.1.0 // indirect
 	github.com/aarondl/json v0.0.0-20221020222930-8b0db17ef1bf // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/google/cel-go v0.20.1 // indirect

go.sum

@@ -4,6 +4,8 @@ connectrpc.com/connect v1.16.2 h1:ybd6y+ls7GOlb7Bh5C8+ghA6SvCBajHwxssO2CGFjqE=
 connectrpc.com/connect v1.16.2/go.mod h1:n2kgwskMHXC+lVqb18wngEpF95ldBHXjZYJussz5FRc=
 connectrpc.com/cors v0.1.0 h1:f3gTXJyDZPrDIZCQ567jxfD9PAIpopHiRDnJRt3QuOQ=
 connectrpc.com/cors v0.1.0/go.mod h1:v8SJZCPfHtGH1zsm+Ttajpozd4cYIUryl4dFB6QEpfg=
+connectrpc.com/validate v0.1.0 h1:r55jirxMK7HO/xZwVHj3w2XkVFarsUM77ZDy367NtH4=
+connectrpc.com/validate v0.1.0/go.mod h1:GU47c9/x/gd+u9wRSPkrQOP46gx2rMN+Wo37EHgI3Ow=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -16,6 +18,7 @@ github.com/aarondl/opt v0.0.0-20240623220848-083f18ab9536 h1:vhpjulzH5Tr4S3uJ3Y/
 github.com/aarondl/opt v0.0.0-20240623220848-083f18ab9536/go.mod h1:l4/5NZtYd/SIohsFhaJQQe+sPOTG22furpZ5FvcYOzk=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM=
 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230512164433-5d1fd1a340c9 h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/bufbuild/protovalidate-go v0.6.3 h1:wxQyzW035zM16Binbaz/nWAzS12dRIXhZdSUWRY7Fv0=

go/api/devices_count.go

@@ -0,0 +1,39 @@
+package api
+
+import (
+	"context"
+
+	"github.com/stephenafamo/bob"
+	"github.com/stephenafamo/bob/dialect/sqlite"
+	"github.com/stephenafamo/bob/dialect/sqlite/dialect"
+	"github.com/stephenafamo/bob/dialect/sqlite/sm"
+	"github.com/tigorlazuardi/bluemage/go/gen/models"
+	device "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1"
+	"github.com/tigorlazuardi/bluemage/go/pkg/errs"
+)
+
+func queryFromCountDevicesRequest(req *device.CountDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) {
+	switch req.Disabled {
+	case device.DisabledFilter_DISABLED_FILTER_TRUE:
+		expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1))
+	case device.DisabledFilter_DISABLED_FILTER_FALSE:
+		expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0))
+	}
+	if req.Search != "" {
+		arg := sqlite.Arg("%" + req.Search + "%")
+		expr = append(expr,
+			sm.Where(
+				models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)),
+			),
+		)
+	}
+	return expr
+}
+
+func (api *API) DevicesCount(ctx context.Context, request *device.CountDevicesRequest) (uint64, error) {
+	count, err := models.Devices.Query(ctx, api.DB, queryFromCountDevicesRequest(request)...).Count()
+	if err != nil {
+		return 0, errs.Wrapw(err, "failed to count devices", "request", request)
+	}
+	return uint64(count), nil
+}

go/api/devices_list.go

@@ -14,7 +14,21 @@ import (
 )
 
 func queryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) {
-	expr = countQueryFromListDeviceRequest(req)
+	switch req.Disabled {
+	case device.DisabledFilter_DISABLED_FILTER_TRUE:
+		expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1))
+	case device.DisabledFilter_DISABLED_FILTER_FALSE:
+		expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0))
+	}
+
+	if req.Search != "" {
+		arg := sqlite.Arg("%" + req.Search + "%")
+		expr = append(expr,
+			sm.Where(
+				models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)),
+			),
+		)
+	}
 
 	if req.Limit > 0 {
 		expr = append(expr, sm.Limit(req.Limit))
@@ -41,46 +55,12 @@ func queryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[
 	return expr
 }
 
-func countQueryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) {
-	switch req.Disabled {
-	case device.DisabledFilter_DISABLED_FILTER_TRUE:
-		expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1))
-	case device.DisabledFilter_DISABLED_FILTER_FALSE:
-		expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0))
-	}
-
-	if req.Search != "" {
-		arg := sqlite.Arg("%" + req.Search + "%")
-		expr = append(expr,
-			sm.Where(
-				models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)),
-			),
-		)
-	}
-	return expr
-}
-
 func (api *API) DevicesList(ctx context.Context, req *device.ListDevicesRequest) (resp *device.ListDevicesResponse, err error) {
 	resp = &device.ListDevicesResponse{}
 	results, err := models.Devices.Query(ctx, api.DB, queryFromListDeviceRequest(req)...).All()
 	if err != nil {
 		return resp, errs.Wrapw(err, "failed to list devices", "request", req)
 	}
-	if req.Disabled == device.DisabledFilter_DISABLED_FILTER_UNSPECIFIED && req.Search == "" {
-		const metricName = "devices.count"
-		metric, err := models.FindMetric(ctx, api.DB, metricName)
-		if err != nil {
-			return resp, errs.Wrapw(err, "failed to find devices count metric", "metric", metricName)
-		}
-		resp.Count = uint64(metric.Value)
-	} else {
-		count, err := models.Devices.Query(ctx, api.DB, countQueryFromListDeviceRequest(req)...).Count()
-		if err != nil {
-			return resp, errs.Wrapw(err, "failed to count query result")
-		}
-		resp.Count = uint64(count)
-	}
-
 	for _, result := range results {
 		resp.Devices = append(resp.Devices, convert.ModelsDeviceToGetDeviceResponse(result))
 	}

go/cmd/bluemage/serve/serve.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"connectrpc.com/connect"
+	"connectrpc.com/validate"
 	sqldblogger "github.com/simukti/sqldb-logger"
 	"github.com/spf13/cobra"
 	"github.com/stephenafamo/bob"
@@ -51,8 +52,16 @@ var Cmd = &cobra.Command{
 			},
 		}
 
+		validationInterceptor, err := validate.NewInterceptor()
+		if err != nil {
+			return errs.Wrap(err, "failed to create validation interceptor")
+		}
+
 		mux := http.NewServeMux()
-		mux.Handle(v1connect.NewDeviceServiceHandler(handler, connect.WithInterceptors(server.LogInterceptor())))
+		mux.Handle(v1connect.NewDeviceServiceHandler(handler, connect.WithInterceptors(
+			validationInterceptor,
+			server.LogInterceptor(),
+		)))
 
 		server := &http.Server{
 			Addr:    ":8080",
@@ -72,7 +81,8 @@ var Cmd = &cobra.Command{
 		if err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return errs.Wrap(err, "failed to serve")
 		}
-		return errors.Join(sqldb.Close())
+		slog.Info("ConnectRPC server stopped")
+		return errors.Join(sqldb.Close(), prettyHandler.Flush())
 	},
 	SilenceUsage: true,
 }

go/converts/converter.go

@@ -14,7 +14,7 @@ import (
 // goverter:extend PtrFloat64ToOmitFloat64
 // goverter:extend PtrInt32ToOmitInt32
 type DeviceConverter interface {
-	// goverter:ignore CreatedAt UpdatedAt
+	// goverter:ignore CreatedAt UpdatedAt R
 	// goverter:map Nsfw NSFW
 	CreateDeviceRequestToModelsDevice(*device.CreateDeviceRequest) *models.Device
 	// goverter:ignore state sizeCache unknownFields

go/server/device_handlers.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"connectrpc.com/connect"
-	"github.com/bufbuild/protovalidate-go"
 	"github.com/tigorlazuardi/bluemage/go/api"
 	"github.com/tigorlazuardi/bluemage/go/gen/converter"
 	device "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1"
@@ -20,19 +19,12 @@ type DeviceHandler struct {
 	v1connect.UnimplementedDeviceServiceHandler
 }
 
-var (
-	convert     converter.DeviceConverterImpl
-	validate, _ = protovalidate.New()
-)
+var convert converter.DeviceConverterImpl
 
 // CreateDevice implements v1connect.DeviceServiceHandler.
 func (d *DeviceHandler) CreateDevice(ctx context.Context, request *connect.Request[device.CreateDeviceRequest]) (*connect.Response[device.CreateDeviceResponse], error) {
-	err := validate.Validate(request.Msg)
-	if err != nil {
-		return nil, connect.NewError(connect.CodeInvalidArgument, err)
-	}
 	dev := convert.CreateDeviceRequestToModelsDevice(request.Msg)
-	dev, err = d.API.DevicesCreate(ctx, dev)
+	dev, err := d.API.DevicesCreate(ctx, dev)
 	if err != nil {
 		return nil, errs.IntoConnectError(err)
 	}
@@ -42,11 +34,6 @@ func (d *DeviceHandler) CreateDevice(ctx context.Context, request *connect.Reque
 
 // GetDevice implements v1connect.DeviceServiceHandler.
 func (d *DeviceHandler) GetDevice(ctx context.Context, request *connect.Request[device.GetDeviceRequest]) (*connect.Response[device.GetDeviceResponse], error) {
-	err := validate.Validate(request.Msg)
-	if err != nil {
-		return nil, connect.NewError(connect.CodeInvalidArgument, err)
-	}
-
 	dev, err := d.API.GetDevice(ctx, request.Msg.Slug)
 	if err != nil {
 		return nil, errs.IntoConnectError(err)
@@ -86,10 +73,6 @@ func (de *DeviceHandler) UpdateDevice(ctx context.Context, request *connect.Requ
 func (de *DeviceHandler) DeviceExists(ctx context.Context, request *connect.Request[device.DeviceExistsRequest]) (*connect.Response[device.DeviceExistsResponse], error) {
 	slug := request.Msg.Slug
 
-	if err := validate.Validate(request.Msg); err != nil {
-		return nil, connect.NewError(connect.CodeInvalidArgument, err)
-	}
-
 	exists, err := de.API.DevicesExist(ctx, slug)
 	if err != nil {
 		return nil, errs.IntoConnectError(err)
@@ -100,3 +83,16 @@ func (de *DeviceHandler) DeviceExists(ctx context.Context, request *connect.Requ
 	}
 	return connect.NewResponse(resp), nil
 }
+
+// CountDevices count the number of devices.
+func (de *DeviceHandler) CountDevices(ctx context.Context, request *connect.Request[device.CountDevicesRequest]) (*connect.Response[device.CountDevicesResponse], error) {
+	count, err := de.API.DevicesCount(ctx, request.Msg)
+	if err != nil {
+		return nil, errs.IntoConnectError(err)
+	}
+
+	resp := &device.CountDevicesResponse{
+		Count: count,
+	}
+	return connect.NewResponse(resp), nil
+}

schemas/migrations/20240804155218_create_metrics_table.sql

@@ -1,21 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-CREATE TABLE metrics (
-    name VARCHAR(255) NOT NULL PRIMARY KEY COLLATE NOCASE,
-    value BIGINT DEFAULT 0 NOT NULL,
-    created_at BIGINT DEFAULT (strftime('%s', 'now')) NOT NULL,
-    updated_at BIGINT DEFAULT (strftime('%s', 'now')) NOT NULL
-);
-
-CREATE UNIQUE INDEX idx_metrics_name ON metrics(name);
-
-CREATE TRIGGER update_metrics_timestamp_after_update AFTER UPDATE ON metrics FOR EACH ROW
-BEGIN
-    UPDATE metrics SET updated_at = (strftime('%s', 'now')) WHERE name = old.name;
-END;
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-DROP TABLE metrics;
--- +goose StatementEnd

schemas/migrations/20240804155710_create_subreddits_table.sql

@@ -12,22 +12,6 @@ CREATE TABLE subreddits (
 );
 
 CREATE UNIQUE INDEX idx_subreddits_name ON subreddits(name);
-
-CREATE TRIGGER subreddits_update_timestamp AFTER UPDATE ON subreddits FOR EACH ROW
-BEGIN
-    UPDATE subreddits SET updated_at = (strftime('%s', 'now')) WHERE name = old.name;
-END;
-
-CREATE TRIGGER subreddits_insert_create_total_image_metrics AFTER INSERT on subreddits FOR EACH ROW
-BEGIN
-    INSERT INTO metrics (name, value)
-    VALUES (CONCAT('subreddits.', NEW.name, '.total_images'), 0);
-END;
-
-CREATE TRIGGER subreddits_delete_total_image_metrics AFTER DELETE on subreddits FOR EACH ROW
-BEGIN
-    DELETE FROM metrics WHERE name = CONCAT('subreddits.', OLD.name, '.total_images');
-END;
 -- +goose StatementEnd
 
 -- +goose Down

schemas/migrations/20240805124501_create_devices_table.sql

@@ -17,8 +17,6 @@ CREATE TABLE devices (
     updated_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now'))
 );
 
-INSERT INTO metrics(name) VALUES ('devices.count');
-
 CREATE UNIQUE INDEX idx_devices_unique_slug ON devices(slug);
 
 CREATE TRIGGER devices_update_timestamp AFTER UPDATE ON devices FOR EACH ROW
@@ -26,22 +24,9 @@ BEGIN
     UPDATE devices SET updated_at = strftime('%s', 'now') WHERE slug = old.slug;
 END;
 
-CREATE TRIGGER devices_insert_create_total_image_metrics AFTER INSERT on devices FOR EACH ROW
-BEGIN
-    INSERT INTO metrics (name, value)
-    VALUES (CONCAT('devices.', NEW.slug, '.total_images'), 0);
-    UPDATE metrics SET value = value + 1 WHERE name = 'devices.count';
-END;
-
-CREATE TRIGGER devices_delete_total_image_metrics AFTER DELETE on devices FOR EACH ROW
-BEGIN
-    DELETE FROM metrics WHERE name = CONCAT('devices.', OLD.slug, '.total_images');
-    UPDATE metrics SET value = value - 1 WHERE name = 'devices.count';
-END;
 -- +goose StatementEnd
 
 -- +goose Down
 -- +goose StatementBegin
 DROP TABLE devices;
-DELETE FROM metrics WHERE name = 'devices.count';
 -- +goose StatementEnd

schemas/migrations/20240808143100_create_images_table.sql

@@ -0,0 +1,42 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE images(
+    id INTEGER PRIMARY KEY,
+    subreddit VARCHAR(255) NOT NULL COLLATE NOCASE,
+    device VARCHAR(250) NOT NULL COLLATE NOCASE,
+    post_title VARCHAR(255) NOT NULL,
+    post_name VARCHAR(255) NOT NULL,
+    post_url VARCHAR(255) NOT NULL,
+    post_created BIGINT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    post_author VARCHAR(50) NOT NULL,
+    post_author_url VARCHAR(255) NOT NULL,
+    image_relative_path VARCHAR(255) NOT NULL,
+    image_original_url VARCHAR(255) NOT NULL,
+    image_height INTEGER NOT NULL DEFAULT 0,
+    image_width INTEGER NOT NULL DEFAULT 0,
+    image_size BIGINT NOT NULL DEFAULT 0,
+    thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '',
+    nsfw INTEGER NOT NULL DEFAULT 0,
+    created_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
+    updated_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')),
+    CONSTRAINT fk_image_subreddit
+        FOREIGN KEY (subreddit)
+        REFERENCES subreddits(name)
+        ON DELETE CASCADE,
+    CONSTRAINT fk_image_devices_slug
+        FOREIGN KEY (device)
+        REFERENCES devices(slug)
+        ON DELETE CASCADE
+);
+
+CREATE INDEX idx_subreddit_images ON images(subreddit);
+CREATE INDEX idx_subreddit_device_images ON images(device, subreddit);
+CREATE INDEX idx_nsfw_images ON images(nsfw);
+CREATE INDEX idx_images_created_at_nsfw ON images(created_at DESC, nsfw);
+CREATE UNIQUE INDEX idx_unique_images_per_device ON images(device, post_name);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE images;
+-- +goose StatementEnd

schemas/proto/device/v1/count.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package device.v1;
+
+import "device/v1/list.proto";
+
+option go_package = "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1";
+
+message CountDevicesRequest {
+  // Limits the counts to devices that have the given name.
+  // case insensitive.
+  //
+  // Ignored if empty.
+  //
+  // default: empty string.
+  string search = 1;
+
+  // disabled limit the counting to devices with the given status.
+  //
+  // If unspecified, devices with either status will be counted.
+  DisabledFilter disabled = 2;
+}
+
+message CountDevicesResponse {
+  uint64 count = 1;
+}

schemas/proto/device/v1/device.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 
 package device.v1;
 
+import "device/v1/count.proto";
 import "device/v1/create.proto";
 import "device/v1/exists.proto";
 import "device/v1/get.proto";
@@ -33,4 +34,7 @@ service DeviceService {
 
   // DeviceExists checks if a device exists in the database.
   rpc DeviceExists(DeviceExistsRequest) returns (DeviceExistsResponse) {}
+
+  // CountDevices count the number of devices.
+  rpc CountDevices(CountDevicesRequest) returns (CountDevicesResponse) {}
 }

schemas/proto/device/v1/list.proto

@@ -47,8 +47,7 @@ message ListDevicesRequest {
 }
 
 message ListDevicesResponse {
-  uint64 count = 1;
-  repeated GetDeviceResponse devices = 2;
+  repeated GetDeviceResponse devices = 1;
 }
 
 enum DisabledFilter {

schemas/proto/device/v1/update.proto

@@ -7,7 +7,7 @@ import "buf/validate/validate.proto";
 option go_package = "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1";
 
 message UpdateDeviceRequest {
-  string slug = 1;
+  string slug = 1 [(buf.validate.field).string.min_len = 1];
   DeviceSetter set = 2;
 }