sqlite: added create images_fts5 table

Makefile

@@ -4,6 +4,7 @@ export PATH := $(shell pwd)/bin:$(shell pwd)/web/node_modules/.bin:$(PATH)
 export GOOSE_DRIVER ?= sqlite3
 export GOOSE_DBSTRING ?= ./go/data.db
 export GOOSE_MIGRATION_DIR ?= schemas/migrations
+export GOFLAGS=-tags=fts5
 
 build: generate-go generate-web
 	go build -o bin/bluemage ./go/cmd/bluemage/main.go

flake.nix

@@ -3,10 +3,19 @@
     nixpkgs.url = "nixpkgs/nixpkgs-unstable";
   };
 
-  outputs = inputs@{ nixpkgs, ... }:
+  outputs = { nixpkgs, ... }:
     let
       system = "x86_64-linux";
-      pkgs = inputs.nixpkgs.legacyPackages.${system};
+
+      # Go-Jet binary in nixos repo is not built with fts5 sqlite enabled
+      overlays = [
+        (final: prev: {
+          go-jet = prev.go-jet.overrideAttrs (oldAttrs: {
+            tags = oldAttrs.tags ++ [ "fts5" ];
+          });
+        })
+      ];
+      pkgs = import nixpkgs { inherit system overlays; };
       goverter = pkgs.buildGoModule rec {
         name = "goverter";
         version = "1.5.0";

go/.air.toml

@@ -1,4 +1,4 @@
-root = "."
+root = ".."
 testdata_dir = "testdata"
 tmp_dir = "tmp"
 
@@ -7,14 +7,14 @@ args_bin = ["serve"]
 bin = "./tmp/main"
 cmd = "go build -o ./tmp/main ./cmd/bluemage"
 delay = 1000
-exclude_dir = ["assets", "tmp", "vendor", "testdata", "gen"]
+exclude_dir = ["assets", "tmp", "vendor", "testdata", "go/gen"]
 exclude_file = []
 exclude_regex = ["_test.go"]
 exclude_unchanged = false
 follow_symlink = false
 full_bin = ""
 include_dir = []
-include_ext = ["go", "tpl", "tmpl", "html"]
+include_ext = ["go", "tpl", "tmpl", "html", "proto"]
 include_file = []
 kill_delay = "5s"
 log = "build-errors.log"

schemas/migrations/20240808143100_create_images_table.sql

@@ -1,7 +1,7 @@
 -- +goose Up
 -- +goose StatementBegin
 CREATE TABLE images(
-    id INTEGER PRIMARY KEY,
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
     subreddit VARCHAR(255) NOT NULL COLLATE NOCASE,
     device VARCHAR(250) NOT NULL COLLATE NOCASE,
     post_title VARCHAR(255) NOT NULL,
@@ -33,6 +33,17 @@ CREATE TABLE images(
         REFERENCES subreddits(cover_image_id)
 );
 
+CREATE VIRTUAL TABLE images_fts5 USING fts5(
+    post_title,
+    post_name,
+    post_author,
+    image_relative_path,
+    image_original_url,
+    thumbnail_relative_path,
+    content='images',
+    content_rowid='id'
+);
+
 CREATE INDEX idx_subreddit_images_blacklisted ON images(subreddit, blacklisted);
 CREATE INDEX idx_subreddit_device_images_blacklisted ON images(device, subreddit, blacklisted);
 CREATE INDEX idx_images_nsfw_blacklisted ON images(nsfw, blacklisted);
@@ -54,9 +65,22 @@ CREATE TRIGGER images_update_subreddit_cover_on_delete AFTER DELETE ON images FO
 BEGIN
     UPDATE subreddits SET cover_image_id = NULL WHERE cover_image_id = old.id;
 END;
+
+CREATE TRIGGER images_update_fts_insert AFTER INSERT ON images FOR EACH ROW
+BEGIN
+    INSERT INTO images_fts5(rowid, post_title, post_name, post_author, image_relative_path, image_original_url, thumbnail_relative_path)
+    VALUES (new.id, new.post_title, new.post_name, new.post_author, new.image_relative_path, new.image_original_url, new.thumbnail_relative_path);
+END;
+
+CREATE TRIGGER images_update_fts_delete AFTER DELETE ON images FOR EACH ROW
+BEGIN
+    DELETE FROM images_fts5 WHERE rowid = old.id;
+END;
+    
 -- +goose StatementEnd
 
 -- +goose Down
 -- +goose StatementBegin
+DROP TABLE images_fts5;
 DROP TABLE images;
 -- +goose StatementEnd

schemas/proto/images/v1/images.proto

@@ -2,61 +2,15 @@ syntax = "proto3";
 
 package images.v1;
 
-import "buf/validate/validate.proto";
+import "images/v1/recently_added.proto";
-import "images/v1/types.proto";
 
 service ImageService {
+  // RecentlyAddedImages returns a list of recently added images.
+  //
+  // The returned images are grouped by device and sorted by the time they were added.
+  //
+  // Use after and before to traverse fetch more images.
+  //
+  // Maximum number of images that can be fetched at a time is 300.
   rpc RecentlyAddedImages(RecentlyAddedImagesRequest) returns (RecentlyAddedImagesResponse) {}
 }
-
-message RecentlyAddedImagesRequest {
-  option (buf.validate.message).cel = {
-    id: "RecentlyAddedImagesRequest.after_cannot_exist_with_before"
-    message: "after and before cannot be set at the same time"
-    expression: "this.after > 0 && this.before > 0"
-  };
-  // subreddits filter the images to be fetched belonging to the given subreddits.
-  //
-  // If empty, images from all subreddits will be fetched.
-  repeated string subreddits = 1;
-  // devices filter the images to be fetched belonging to the given devices.
-  //
-  // If empty, images from all devices will be fetched.
-  repeated string devices = 2;
-  // limit limits the number of images to be fetched.
-  //
-  // if 0 (or not set), the default limit is 100.
-  //
-  // if set, the maximum limit is clamped to 1000.
-  int64 limit = 3;
-  // after lists the image after the given timestamp.
-  //
-  // using after will return images that are created after the given timestamp.
-  //
-  // cannot be set (value > 0) with before (value > 0).
-  int64 after = 4;
-  // before lists the image before the given timestamp.
-  //
-  // using before will return images that are created before the given timestamp.
-  //
-  // cannot be set (value > 0) with after (value > 0).
-  int64 before = 5;
-}
-
-message RecentlyAddedImagesResponse {
-  repeated GroupedByDeviceImages groups = 1;
-  // after is given to the client to be used as after in the next request.
-  //
-  // Given if the request has after value set.
-  optional int64 after = 2;
-
-  // before is given to the client to be used as before in the next request.
-  //
-  // Given if the request has before value set.
-  optional int64 before = 3;
-}
-
-message GroupedByDeviceImages {
-  string device = 1;
-  repeated Image images = 2;
-}

schemas/proto/images/v1/list_by_subreddit.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package images.v1;
+
+import "buf/validate/validate.proto";
+import "images/v1/types.proto";
+
+message ListBySubredditRequest {
+  string subreddit = 1 [(buf.validate.field).string.min_len = 1];
+}
+
+message ListBySubredditResponse {
+  repeated GroupedByDeviceImages devices = 1;
+  optional int64 after = 2;
+  optional int64 before = 3;
+}

schemas/proto/images/v1/recently_added.proto

@@ -0,0 +1,56 @@
+syntax = "proto3";
+
+package images.v1;
+
+import "buf/validate/validate.proto";
+import "images/v1/types.proto";
+
+message RecentlyAddedImagesRequest {
+  option (buf.validate.message).cel = {
+    id: "RecentlyAddedImagesRequest.after_cannot_exist_with_before"
+    message: "after and before cannot be set at the same time"
+    expression: "this.after > 0 && this.before > 0"
+  };
+  // subreddits filter the images to be fetched belonging to the given subreddits.
+  //
+  // If empty, images from all subreddits will be fetched.
+  repeated string subreddits = 1;
+  // devices filter the images to be fetched belonging to the given devices.
+  //
+  // If empty, images from all devices will be fetched.
+  repeated string devices = 2;
+  // limit limits the number of images to be fetched.
+  //
+  // if 0 (or not set), the default limit is 100.
+  //
+  // if set, the maximum limit is clamped to 300.
+  int64 limit = 3 [(buf.validate.field).int64.gte = 0];
+  // after lists the image after the given timestamp.
+  //
+  // using after will return images that are created after the given timestamp.
+  //
+  // cannot be set (value > 0) with before (value > 0).
+  int64 after = 4;
+  // before lists the image before the given timestamp.
+  //
+  // using before will return images that are created before the given timestamp.
+  //
+  // cannot be set (value > 0) with after (value > 0).
+  int64 before = 5;
+}
+
+message RecentlyAddedImagesResponse {
+  // groups are images grouped by devices.
+  //
+  // devices are sorted alphabetically by it's NAME not slug.
+  repeated GroupedBySubredditDevices groups = 1;
+  // after is a unix epoch timestamp given to the client to be used as after in the next request.
+  //
+  // Given if the request has after value set and there are more images to be fetched.
+  optional int64 after = 2;
+
+  // before is a unix epoch timestamp given to the client to be used as before in the next request.
+  //
+  // Given if the request has before value set and there are more images to be fetched.
+  optional int64 before = 3;
+}

schemas/proto/images/v1/types.proto

@@ -17,3 +17,14 @@ message Image {
   uint32 image_width = 12;
   int64 image_size = 13;
 }
+
+message GroupedByDeviceImages {
+  string slug = 1;
+  string name = 2;
+  repeated Image images = 3;
+}
+
+message GroupedBySubredditDevices {
+  string subreddit = 1;
+  repeated GroupedByDeviceImages devices = 2;
+}