Compare commits

...

2 commits

8 changed files with 131 additions and 60 deletions

View file

@ -4,6 +4,7 @@ export PATH := $(shell pwd)/bin:$(shell pwd)/web/node_modules/.bin:$(PATH)
export GOOSE_DRIVER ?= sqlite3 export GOOSE_DRIVER ?= sqlite3
export GOOSE_DBSTRING ?= ./go/data.db export GOOSE_DBSTRING ?= ./go/data.db
export GOOSE_MIGRATION_DIR ?= schemas/migrations export GOOSE_MIGRATION_DIR ?= schemas/migrations
export GOFLAGS=-tags=fts5
build: generate-go generate-web build: generate-go generate-web
go build -o bin/bluemage ./go/cmd/bluemage/main.go go build -o bin/bluemage ./go/cmd/bluemage/main.go

View file

@ -3,10 +3,19 @@
nixpkgs.url = "nixpkgs/nixpkgs-unstable"; nixpkgs.url = "nixpkgs/nixpkgs-unstable";
}; };
outputs = inputs@{ nixpkgs, ... }: outputs = { nixpkgs, ... }:
let let
system = "x86_64-linux"; system = "x86_64-linux";
pkgs = inputs.nixpkgs.legacyPackages.${system};
# Go-Jet binary in nixos repo is not built with fts5 sqlite enabled
overlays = [
(final: prev: {
go-jet = prev.go-jet.overrideAttrs (oldAttrs: {
tags = oldAttrs.tags ++ [ "fts5" ];
});
})
];
pkgs = import nixpkgs { inherit system overlays; };
goverter = pkgs.buildGoModule rec { goverter = pkgs.buildGoModule rec {
name = "goverter"; name = "goverter";
version = "1.5.0"; version = "1.5.0";

View file

@ -1,4 +1,4 @@
root = "." root = ".."
testdata_dir = "testdata" testdata_dir = "testdata"
tmp_dir = "tmp" tmp_dir = "tmp"
@ -7,14 +7,14 @@ args_bin = ["serve"]
bin = "./tmp/main" bin = "./tmp/main"
cmd = "go build -o ./tmp/main ./cmd/bluemage" cmd = "go build -o ./tmp/main ./cmd/bluemage"
delay = 1000 delay = 1000
exclude_dir = ["assets", "tmp", "vendor", "testdata", "gen"] exclude_dir = ["assets", "tmp", "vendor", "testdata", "go/gen"]
exclude_file = [] exclude_file = []
exclude_regex = ["_test.go"] exclude_regex = ["_test.go"]
exclude_unchanged = false exclude_unchanged = false
follow_symlink = false follow_symlink = false
full_bin = "" full_bin = ""
include_dir = [] include_dir = []
include_ext = ["go", "tpl", "tmpl", "html"] include_ext = ["go", "tpl", "tmpl", "html", "proto"]
include_file = [] include_file = []
kill_delay = "5s" kill_delay = "5s"
log = "build-errors.log" log = "build-errors.log"

View file

@ -1,7 +1,7 @@
-- +goose Up -- +goose Up
-- +goose StatementBegin -- +goose StatementBegin
CREATE TABLE images( CREATE TABLE images(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY AUTOINCREMENT,
subreddit VARCHAR(255) NOT NULL COLLATE NOCASE, subreddit VARCHAR(255) NOT NULL COLLATE NOCASE,
device VARCHAR(250) NOT NULL COLLATE NOCASE, device VARCHAR(250) NOT NULL COLLATE NOCASE,
post_title VARCHAR(255) NOT NULL, post_title VARCHAR(255) NOT NULL,
@ -33,6 +33,17 @@ CREATE TABLE images(
REFERENCES subreddits(cover_image_id) REFERENCES subreddits(cover_image_id)
); );
CREATE VIRTUAL TABLE images_fts5 USING fts5(
post_title,
post_name,
post_author,
image_relative_path,
image_original_url,
thumbnail_relative_path,
content='images',
content_rowid='id'
);
CREATE INDEX idx_subreddit_images_blacklisted ON images(subreddit, blacklisted); CREATE INDEX idx_subreddit_images_blacklisted ON images(subreddit, blacklisted);
CREATE INDEX idx_subreddit_device_images_blacklisted ON images(device, subreddit, blacklisted); CREATE INDEX idx_subreddit_device_images_blacklisted ON images(device, subreddit, blacklisted);
CREATE INDEX idx_images_nsfw_blacklisted ON images(nsfw, blacklisted); CREATE INDEX idx_images_nsfw_blacklisted ON images(nsfw, blacklisted);
@ -54,9 +65,22 @@ CREATE TRIGGER images_update_subreddit_cover_on_delete AFTER DELETE ON images FO
BEGIN BEGIN
UPDATE subreddits SET cover_image_id = NULL WHERE cover_image_id = old.id; UPDATE subreddits SET cover_image_id = NULL WHERE cover_image_id = old.id;
END; END;
CREATE TRIGGER images_update_fts_insert AFTER INSERT ON images FOR EACH ROW
BEGIN
INSERT INTO images_fts5(rowid, post_title, post_name, post_author, image_relative_path, image_original_url, thumbnail_relative_path)
VALUES (new.id, new.post_title, new.post_name, new.post_author, new.image_relative_path, new.image_original_url, new.thumbnail_relative_path);
END;
CREATE TRIGGER images_update_fts_delete AFTER DELETE ON images FOR EACH ROW
BEGIN
DELETE FROM images_fts5 WHERE rowid = old.id;
END;
-- +goose StatementEnd -- +goose StatementEnd
-- +goose Down -- +goose Down
-- +goose StatementBegin -- +goose StatementBegin
DROP TABLE images_fts5;
DROP TABLE images; DROP TABLE images;
-- +goose StatementEnd -- +goose StatementEnd

View file

@ -2,61 +2,15 @@ syntax = "proto3";
package images.v1; package images.v1;
import "buf/validate/validate.proto"; import "images/v1/recently_added.proto";
import "images/v1/types.proto";
service ImageService { service ImageService {
// RecentlyAddedImages returns a list of recently added images.
//
// The returned images are grouped by device and sorted by the time they were added.
//
// Use after and before to traverse fetch more images.
//
// Maximum number of images that can be fetched at a time is 300.
rpc RecentlyAddedImages(RecentlyAddedImagesRequest) returns (RecentlyAddedImagesResponse) {} rpc RecentlyAddedImages(RecentlyAddedImagesRequest) returns (RecentlyAddedImagesResponse) {}
} }
message RecentlyAddedImagesRequest {
option (buf.validate.message).cel = {
id: "RecentlyAddedImagesRequest.after_cannot_exist_with_before"
message: "after and before cannot be set at the same time"
expression: "this.after > 0 && this.before > 0"
};
// subreddits filter the images to be fetched belonging to the given subreddits.
//
// If empty, images from all subreddits will be fetched.
repeated string subreddits = 1;
// devices filter the images to be fetched belonging to the given devices.
//
// If empty, images from all devices will be fetched.
repeated string devices = 2;
// limit limits the number of images to be fetched.
//
// if 0 (or not set), the default limit is 100.
//
// if set, the maximum limit is clamped to 1000.
int64 limit = 3;
// after lists the image after the given timestamp.
//
// using after will return images that are created after the given timestamp.
//
// cannot be set (value > 0) with before (value > 0).
int64 after = 4;
// before lists the image before the given timestamp.
//
// using before will return images that are created before the given timestamp.
//
// cannot be set (value > 0) with after (value > 0).
int64 before = 5;
}
message RecentlyAddedImagesResponse {
repeated GroupedByDeviceImages groups = 1;
// after is given to the client to be used as after in the next request.
//
// Given if the request has after value set.
optional int64 after = 2;
// before is given to the client to be used as before in the next request.
//
// Given if the request has before value set.
optional int64 before = 3;
}
message GroupedByDeviceImages {
string device = 1;
repeated Image images = 2;
}

View file

@ -0,0 +1,16 @@
syntax = "proto3";
package images.v1;
import "buf/validate/validate.proto";
import "images/v1/types.proto";
message ListBySubredditRequest {
string subreddit = 1 [(buf.validate.field).string.min_len = 1];
}
message ListBySubredditResponse {
repeated GroupedByDeviceImages devices = 1;
optional int64 after = 2;
optional int64 before = 3;
}

View file

@ -0,0 +1,56 @@
syntax = "proto3";
package images.v1;
import "buf/validate/validate.proto";
import "images/v1/types.proto";
message RecentlyAddedImagesRequest {
option (buf.validate.message).cel = {
id: "RecentlyAddedImagesRequest.after_cannot_exist_with_before"
message: "after and before cannot be set at the same time"
expression: "this.after > 0 && this.before > 0"
};
// subreddits filter the images to be fetched belonging to the given subreddits.
//
// If empty, images from all subreddits will be fetched.
repeated string subreddits = 1;
// devices filter the images to be fetched belonging to the given devices.
//
// If empty, images from all devices will be fetched.
repeated string devices = 2;
// limit limits the number of images to be fetched.
//
// if 0 (or not set), the default limit is 100.
//
// if set, the maximum limit is clamped to 300.
int64 limit = 3 [(buf.validate.field).int64.gte = 0];
// after lists the image after the given timestamp.
//
// using after will return images that are created after the given timestamp.
//
// cannot be set (value > 0) with before (value > 0).
int64 after = 4;
// before lists the image before the given timestamp.
//
// using before will return images that are created before the given timestamp.
//
// cannot be set (value > 0) with after (value > 0).
int64 before = 5;
}
message RecentlyAddedImagesResponse {
// groups are images grouped by devices.
//
// devices are sorted alphabetically by it's NAME not slug.
repeated GroupedBySubredditDevices groups = 1;
// after is a unix epoch timestamp given to the client to be used as after in the next request.
//
// Given if the request has after value set and there are more images to be fetched.
optional int64 after = 2;
// before is a unix epoch timestamp given to the client to be used as before in the next request.
//
// Given if the request has before value set and there are more images to be fetched.
optional int64 before = 3;
}

View file

@ -17,3 +17,14 @@ message Image {
uint32 image_width = 12; uint32 image_width = 12;
int64 image_size = 13; int64 image_size = 13;
} }
message GroupedByDeviceImages {
string slug = 1;
string name = 2;
repeated Image images = 3;
}
message GroupedBySubredditDevices {
string subreddit = 1;
repeated GroupedByDeviceImages devices = 2;
}