From ab86ff19a500512c57088e043c1bc246c4dc1ab4 Mon Sep 17 00:00:00 2001 From: Tigor Hutasuhut Date: Thu, 8 Aug 2024 22:51:53 +0700 Subject: [PATCH] devices: added count devices endpoint --- go/api/devices_count.go | 39 +++++++++++++++ go/api/devices_list.go | 50 ++++++------------- go/cmd/bluemage/serve/serve.go | 3 +- go/converts/converter.go | 2 +- go/server/device_handlers.go | 13 +++++ .../20240804155218_create_metrics_table.sql | 21 -------- ...20240804155710_create_subreddits_table.sql | 16 ------ .../20240805124501_create_devices_table.sql | 15 ------ .../20240808143100_create_images_table.sql | 42 ++++++++++++++++ schemas/proto/device/v1/count.proto | 26 ++++++++++ schemas/proto/device/v1/device.proto | 4 ++ schemas/proto/device/v1/list.proto | 3 +- 12 files changed, 143 insertions(+), 91 deletions(-) create mode 100644 go/api/devices_count.go delete mode 100644 schemas/migrations/20240804155218_create_metrics_table.sql create mode 100644 schemas/migrations/20240808143100_create_images_table.sql create mode 100644 schemas/proto/device/v1/count.proto diff --git a/go/api/devices_count.go b/go/api/devices_count.go new file mode 100644 index 0000000..14cdeb1 --- /dev/null +++ b/go/api/devices_count.go @@ -0,0 +1,39 @@ +package api + +import ( + "context" + + "github.com/stephenafamo/bob" + "github.com/stephenafamo/bob/dialect/sqlite" + "github.com/stephenafamo/bob/dialect/sqlite/dialect" + "github.com/stephenafamo/bob/dialect/sqlite/sm" + "github.com/tigorlazuardi/bluemage/go/gen/models" + device "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1" + "github.com/tigorlazuardi/bluemage/go/pkg/errs" +) + +func queryFromCountDevicesRequest(req *device.CountDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) { + switch req.Disabled { + case device.DisabledFilter_DISABLED_FILTER_TRUE: + expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1)) + case device.DisabledFilter_DISABLED_FILTER_FALSE: + expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0)) + } + if req.Search != "" { + arg := sqlite.Arg("%" + req.Search + "%") + expr = append(expr, + sm.Where( + models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)), + ), + ) + } + return expr +} + +func (api *API) DevicesCount(ctx context.Context, request *device.CountDevicesRequest) (uint64, error) { + count, err := models.Devices.Query(ctx, api.DB, queryFromCountDevicesRequest(request)...).Count() + if err != nil { + return 0, errs.Wrapw(err, "failed to count devices", "request", request) + } + return uint64(count), nil +} diff --git a/go/api/devices_list.go b/go/api/devices_list.go index 8a0b2a5..f5a289a 100644 --- a/go/api/devices_list.go +++ b/go/api/devices_list.go @@ -14,7 +14,21 @@ import ( ) func queryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) { - expr = countQueryFromListDeviceRequest(req) + switch req.Disabled { + case device.DisabledFilter_DISABLED_FILTER_TRUE: + expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1)) + case device.DisabledFilter_DISABLED_FILTER_FALSE: + expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0)) + } + + if req.Search != "" { + arg := sqlite.Arg("%" + req.Search + "%") + expr = append(expr, + sm.Where( + models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)), + ), + ) + } if req.Limit > 0 { expr = append(expr, sm.Limit(req.Limit)) @@ -41,46 +55,12 @@ func queryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[ return expr } -func countQueryFromListDeviceRequest(req *device.ListDevicesRequest) (expr []bob.Mod[*dialect.SelectQuery]) { - switch req.Disabled { - case device.DisabledFilter_DISABLED_FILTER_TRUE: - expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(1)) - case device.DisabledFilter_DISABLED_FILTER_FALSE: - expr = append(expr, models.SelectWhere.Devices.Disabled.EQ(0)) - } - - if req.Search != "" { - arg := sqlite.Arg("%" + req.Search + "%") - expr = append(expr, - sm.Where( - models.DeviceColumns.Name.Like(arg).Or(models.DeviceColumns.Slug.Like(arg)), - ), - ) - } - return expr -} - func (api *API) DevicesList(ctx context.Context, req *device.ListDevicesRequest) (resp *device.ListDevicesResponse, err error) { resp = &device.ListDevicesResponse{} results, err := models.Devices.Query(ctx, api.DB, queryFromListDeviceRequest(req)...).All() if err != nil { return resp, errs.Wrapw(err, "failed to list devices", "request", req) } - if req.Disabled == device.DisabledFilter_DISABLED_FILTER_UNSPECIFIED && req.Search == "" { - const metricName = "devices.count" - metric, err := models.FindMetric(ctx, api.DB, metricName) - if err != nil { - return resp, errs.Wrapw(err, "failed to find devices count metric", "metric", metricName) - } - resp.Count = uint64(metric.Value) - } else { - count, err := models.Devices.Query(ctx, api.DB, countQueryFromListDeviceRequest(req)...).Count() - if err != nil { - return resp, errs.Wrapw(err, "failed to count query result") - } - resp.Count = uint64(count) - } - for _, result := range results { resp.Devices = append(resp.Devices, convert.ModelsDeviceToGetDeviceResponse(result)) } diff --git a/go/cmd/bluemage/serve/serve.go b/go/cmd/bluemage/serve/serve.go index 5e77f77..055c488 100644 --- a/go/cmd/bluemage/serve/serve.go +++ b/go/cmd/bluemage/serve/serve.go @@ -81,7 +81,8 @@ var Cmd = &cobra.Command{ if err != nil && !errors.Is(err, http.ErrServerClosed) { return errs.Wrap(err, "failed to serve") } - return errors.Join(sqldb.Close()) + slog.Info("ConnectRPC server stopped") + return errors.Join(sqldb.Close(), prettyHandler.Flush()) }, SilenceUsage: true, } diff --git a/go/converts/converter.go b/go/converts/converter.go index ddca75f..2ce9444 100644 --- a/go/converts/converter.go +++ b/go/converts/converter.go @@ -14,7 +14,7 @@ import ( // goverter:extend PtrFloat64ToOmitFloat64 // goverter:extend PtrInt32ToOmitInt32 type DeviceConverter interface { - // goverter:ignore CreatedAt UpdatedAt + // goverter:ignore CreatedAt UpdatedAt R // goverter:map Nsfw NSFW CreateDeviceRequestToModelsDevice(*device.CreateDeviceRequest) *models.Device // goverter:ignore state sizeCache unknownFields diff --git a/go/server/device_handlers.go b/go/server/device_handlers.go index 0245113..e012aee 100644 --- a/go/server/device_handlers.go +++ b/go/server/device_handlers.go @@ -83,3 +83,16 @@ func (de *DeviceHandler) DeviceExists(ctx context.Context, request *connect.Requ } return connect.NewResponse(resp), nil } + +// CountDevices count the number of devices. +func (de *DeviceHandler) CountDevices(ctx context.Context, request *connect.Request[device.CountDevicesRequest]) (*connect.Response[device.CountDevicesResponse], error) { + count, err := de.API.DevicesCount(ctx, request.Msg) + if err != nil { + return nil, errs.IntoConnectError(err) + } + + resp := &device.CountDevicesResponse{ + Count: count, + } + return connect.NewResponse(resp), nil +} diff --git a/schemas/migrations/20240804155218_create_metrics_table.sql b/schemas/migrations/20240804155218_create_metrics_table.sql deleted file mode 100644 index 95a8eb7..0000000 --- a/schemas/migrations/20240804155218_create_metrics_table.sql +++ /dev/null @@ -1,21 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE metrics ( - name VARCHAR(255) NOT NULL PRIMARY KEY COLLATE NOCASE, - value BIGINT DEFAULT 0 NOT NULL, - created_at BIGINT DEFAULT (strftime('%s', 'now')) NOT NULL, - updated_at BIGINT DEFAULT (strftime('%s', 'now')) NOT NULL -); - -CREATE UNIQUE INDEX idx_metrics_name ON metrics(name); - -CREATE TRIGGER update_metrics_timestamp_after_update AFTER UPDATE ON metrics FOR EACH ROW -BEGIN - UPDATE metrics SET updated_at = (strftime('%s', 'now')) WHERE name = old.name; -END; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -DROP TABLE metrics; --- +goose StatementEnd diff --git a/schemas/migrations/20240804155710_create_subreddits_table.sql b/schemas/migrations/20240804155710_create_subreddits_table.sql index 417d874..1b0f311 100644 --- a/schemas/migrations/20240804155710_create_subreddits_table.sql +++ b/schemas/migrations/20240804155710_create_subreddits_table.sql @@ -12,22 +12,6 @@ CREATE TABLE subreddits ( ); CREATE UNIQUE INDEX idx_subreddits_name ON subreddits(name); - -CREATE TRIGGER subreddits_update_timestamp AFTER UPDATE ON subreddits FOR EACH ROW -BEGIN - UPDATE subreddits SET updated_at = (strftime('%s', 'now')) WHERE name = old.name; -END; - -CREATE TRIGGER subreddits_insert_create_total_image_metrics AFTER INSERT on subreddits FOR EACH ROW -BEGIN - INSERT INTO metrics (name, value) - VALUES (CONCAT('subreddits.', NEW.name, '.total_images'), 0); -END; - -CREATE TRIGGER subreddits_delete_total_image_metrics AFTER DELETE on subreddits FOR EACH ROW -BEGIN - DELETE FROM metrics WHERE name = CONCAT('subreddits.', OLD.name, '.total_images'); -END; -- +goose StatementEnd -- +goose Down diff --git a/schemas/migrations/20240805124501_create_devices_table.sql b/schemas/migrations/20240805124501_create_devices_table.sql index 6bac0d5..ba42b18 100644 --- a/schemas/migrations/20240805124501_create_devices_table.sql +++ b/schemas/migrations/20240805124501_create_devices_table.sql @@ -17,8 +17,6 @@ CREATE TABLE devices ( updated_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')) ); -INSERT INTO metrics(name) VALUES ('devices.count'); - CREATE UNIQUE INDEX idx_devices_unique_slug ON devices(slug); CREATE TRIGGER devices_update_timestamp AFTER UPDATE ON devices FOR EACH ROW @@ -26,22 +24,9 @@ BEGIN UPDATE devices SET updated_at = strftime('%s', 'now') WHERE slug = old.slug; END; -CREATE TRIGGER devices_insert_create_total_image_metrics AFTER INSERT on devices FOR EACH ROW -BEGIN - INSERT INTO metrics (name, value) - VALUES (CONCAT('devices.', NEW.slug, '.total_images'), 0); - UPDATE metrics SET value = value + 1 WHERE name = 'devices.count'; -END; - -CREATE TRIGGER devices_delete_total_image_metrics AFTER DELETE on devices FOR EACH ROW -BEGIN - DELETE FROM metrics WHERE name = CONCAT('devices.', OLD.slug, '.total_images'); - UPDATE metrics SET value = value - 1 WHERE name = 'devices.count'; -END; -- +goose StatementEnd -- +goose Down -- +goose StatementBegin DROP TABLE devices; -DELETE FROM metrics WHERE name = 'devices.count'; -- +goose StatementEnd diff --git a/schemas/migrations/20240808143100_create_images_table.sql b/schemas/migrations/20240808143100_create_images_table.sql new file mode 100644 index 0000000..a02a195 --- /dev/null +++ b/schemas/migrations/20240808143100_create_images_table.sql @@ -0,0 +1,42 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE images( + id INTEGER PRIMARY KEY, + subreddit VARCHAR(255) NOT NULL COLLATE NOCASE, + device VARCHAR(250) NOT NULL COLLATE NOCASE, + post_title VARCHAR(255) NOT NULL, + post_name VARCHAR(255) NOT NULL, + post_url VARCHAR(255) NOT NULL, + post_created BIGINT NOT NULL DEFAULT CURRENT_TIMESTAMP, + post_author VARCHAR(50) NOT NULL, + post_author_url VARCHAR(255) NOT NULL, + image_relative_path VARCHAR(255) NOT NULL, + image_original_url VARCHAR(255) NOT NULL, + image_height INTEGER NOT NULL DEFAULT 0, + image_width INTEGER NOT NULL DEFAULT 0, + image_size BIGINT NOT NULL DEFAULT 0, + thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '', + nsfw INTEGER NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')), + updated_at BIGINT NOT NULL DEFAULT (strftime('%s', 'now')), + CONSTRAINT fk_image_subreddit + FOREIGN KEY (subreddit) + REFERENCES subreddits(name) + ON DELETE CASCADE, + CONSTRAINT fk_image_devices_slug + FOREIGN KEY (device) + REFERENCES devices(slug) + ON DELETE CASCADE +); + +CREATE INDEX idx_subreddit_images ON images(subreddit); +CREATE INDEX idx_subreddit_device_images ON images(device, subreddit); +CREATE INDEX idx_nsfw_images ON images(nsfw); +CREATE INDEX idx_images_created_at_nsfw ON images(created_at DESC, nsfw); +CREATE UNIQUE INDEX idx_unique_images_per_device ON images(device, post_name); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE images; +-- +goose StatementEnd diff --git a/schemas/proto/device/v1/count.proto b/schemas/proto/device/v1/count.proto new file mode 100644 index 0000000..a383331 --- /dev/null +++ b/schemas/proto/device/v1/count.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package device.v1; + +import "device/v1/list.proto"; + +option go_package = "github.com/tigorlazuardi/bluemage/go/gen/proto/device/v1"; + +message CountDevicesRequest { + // Limits the counts to devices that have the given name. + // case insensitive. + // + // Ignored if empty. + // + // default: empty string. + string search = 1; + + // disabled limit the counting to devices with the given status. + // + // If unspecified, devices with either status will be counted. + DisabledFilter disabled = 2; +} + +message CountDevicesResponse { + uint64 count = 1; +} diff --git a/schemas/proto/device/v1/device.proto b/schemas/proto/device/v1/device.proto index 8153b29..1e8b365 100644 --- a/schemas/proto/device/v1/device.proto +++ b/schemas/proto/device/v1/device.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package device.v1; +import "device/v1/count.proto"; import "device/v1/create.proto"; import "device/v1/exists.proto"; import "device/v1/get.proto"; @@ -33,4 +34,7 @@ service DeviceService { // DeviceExists checks if a device exists in the database. rpc DeviceExists(DeviceExistsRequest) returns (DeviceExistsResponse) {} + + // CountDevices count the number of devices. + rpc CountDevices(CountDevicesRequest) returns (CountDevicesResponse) {} } diff --git a/schemas/proto/device/v1/list.proto b/schemas/proto/device/v1/list.proto index ca661b3..64e036b 100644 --- a/schemas/proto/device/v1/list.proto +++ b/schemas/proto/device/v1/list.proto @@ -47,8 +47,7 @@ message ListDevicesRequest { } message ListDevicesResponse { - uint64 count = 1; - repeated GetDeviceResponse devices = 2; + repeated GetDeviceResponse devices = 1; } enum DisabledFilter {