refactor: simpler data columns

api/devices_update.go

@@ -9,11 +9,11 @@ import (
 	"github.com/tigorlazuardi/redmage/pkg/errs"
 )
 
-func (api *API) DevicesUpdate(ctx context.Context, id int, update *models.DeviceSetter) (device *models.Device, err error) {
+func (api *API) DevicesUpdate(ctx context.Context, slug string, update *models.DeviceSetter) (device *models.Device, err error) {
 	ctx, span := tracer.Start(ctx, "*API.DevicesUpdate")
 	defer span.End()
 
-	device = &models.Device{ID: int32(id)}
+	device = &models.Device{Slug: slug}
 
 	err = models.Devices.Update(ctx, api.db, update, device)
 	if err != nil {
@@ -23,11 +23,11 @@ func (api *API) DevicesUpdate(ctx context.Context, id int, update *models.Device
 				return device, errs.Wrapw(err, "a device with the same slug id already exists").Code(409)
 			}
 		}
-		return device, errs.Wrapw(err, "failed to update device", "id", id, "values", update)
+		return device, errs.Wrapw(err, "failed to update device", "slug", slug, "values", update)
 	}
 
 	if err := device.Reload(ctx, api.db); err != nil {
-		return device, errs.Wrapw(err, "failed to reload device", "id", id)
+		return device, errs.Wrapw(err, "failed to reload device", "slug", slug)
 	}
 
 	return device, nil

api/download_subreddit_images.go

@@ -100,11 +100,11 @@ func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.List
 		if !post.IsImagePost() {
 			continue
 		}
-		devices := api.getDevicesThatAcceptPost(ctx, post, devices)
+		acceptedDevices := api.getDevicesThatAcceptPost(ctx, post, devices)
-		if len(devices) == 0 {
+		if len(acceptedDevices) == 0 {
 			continue
 		}
-		log.New(ctx).Debug("downloading image", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", devices)
+		log.New(ctx).Debug("downloading image", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", acceptedDevices)
 		wg.Add(1)
 		api.imageSemaphore <- struct{}{}
 		go func(ctx context.Context, post reddit.Post) {
@@ -113,7 +113,15 @@ func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.List
 				wg.Done()
 			}()
 
-			if err := api.downloadSubredditImage(ctx, post, subreddit, devices); err != nil {
+			if imageFile := api.findImageFileForDevices(ctx, post, devices); imageFile != nil {
+				err := api.saveImageToFSAndDatabase(ctx, imageFile, subreddit, post, acceptedDevices)
+				if err != nil {
+					log.New(ctx).Err(err).Error("failed to download subreddit image")
+				}
+				return
+			}
+
+			if err := api.downloadSubredditImage(ctx, post, subreddit, acceptedDevices); err != nil {
 				log.New(ctx).Err(err).Error("failed to download subreddit image")
 			}
 		}(ctx, post)
@@ -174,13 +182,21 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su
 		return errs.Wrapw(err, "failed to encode thumbnail file to jpeg", "filename", thumbnailPath)
 	}
 
+	return api.saveImageToFSAndDatabase(ctx, tmpImageFile, subreddit, post, devices)
+}
+
+func (api *API) saveImageToFSAndDatabase(ctx context.Context, image io.ReadCloser, subreddit *models.Subreddit, post reddit.Post, devices models.DeviceSlice) (err error) {
+	ctx, span := tracer.Start(ctx, "*API.saveImageToFSAndDatabase")
+	defer span.End()
+	defer image.Close()
+
 	w, close, err := api.createDeviceImageWriters(post, devices)
 	if err != nil {
 		return errs.Wrapw(err, "failed to create image files")
 	}
 	log.New(ctx).Debug("saving image files", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", devices)
 	defer close()
-	_, err = io.Copy(w, tmpImageFile)
+	_, err = io.Copy(w, image)
 	if err != nil {
 		return errs.Wrapw(err, "failed to save image files")
 	}
@@ -192,20 +208,27 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su
 		if post.IsNSFW() {
 			nsfw = 1
 		}
+		width, height := post.GetImageSize()
+		var size int64
+		if fi, err := os.Stat(post.GetImageTargetPath(api.config, device)); err == nil {
+			size = fi.Size()
+		}
+
 		many = append(many, &models.ImageSetter{
-			SubredditID:           omit.From(subreddit.ID),
+			Subreddit:             omit.From(subreddit.Name),
-			DeviceID:              omit.From(device.ID),
+			Device:                omit.From(device.Slug),
-			Title:                 omit.From(post.GetTitle()),
+			PostTitle:             omit.From(post.GetTitle()),
-			PostID:                omit.From(post.GetID()),
 			PostURL:               omit.From(post.GetPostURL()),
 			PostCreated:           omit.From(post.GetCreated().Unix()),
 			PostName:              omit.From(post.GetName()),
-			Poster:                omit.From(post.GetAuthor()),
+			PostAuthor:            omit.From(post.GetAuthor()),
-			PosterURL:             omit.From(post.GetAuthorURL()),
+			PostAuthorURL:         omit.From(post.GetAuthorURL()),
+			ImageWidth:            omit.From(int32(width)),
+			ImageHeight:           omit.From(int32(height)),
+			ImageSize:             omit.From(size),
 			ImageRelativePath:     omit.From(post.GetImageRelativePath(device)),
 			ThumbnailRelativePath: omit.From(post.GetThumbnailRelativePath()),
 			ImageOriginalURL:      omit.From(post.GetImageURL()),
-			ThumbnailOriginalURL:  omit.From(post.GetThumbnailURL()),
 			NSFW:                  omit.From(nsfw),
 			CreatedAt:             omit.From(now.Unix()),
 			UpdatedAt:             omit.From(now.Unix()),
@@ -217,7 +240,6 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su
 	if err != nil {
 		return errs.Wrapw(err, "failed to insert images to database", "params", many)
 	}
-
 	return nil
 }
 
@@ -265,20 +287,22 @@ func (api *API) createDeviceImageWriters(post reddit.Post, devices models.Device
 
 func (api *API) getDevicesThatAcceptPost(ctx context.Context, post reddit.Post, devices models.DeviceSlice) (devs models.DeviceSlice) {
 	for _, device := range devices {
-		if shouldDownloadPostForDevice(post, device) && !api.isImageExists(ctx, post, device) {
+		if shouldDownloadPostForDevice(post, device) && !api.isImageEntryExists(ctx, post, device) {
 			devs = append(devs, device)
 		}
 	}
 	return devs
 }
 
-func (api *API) isImageExists(ctx context.Context, post reddit.Post, device *models.Device) (found bool) {
+// isImageEntryExists checks if the image entry already exists in the database and
+// the image file actually exists in the filesystem.
+func (api *API) isImageEntryExists(ctx context.Context, post reddit.Post, device *models.Device) (found bool) {
 	ctx, span := tracer.Start(ctx, "*API.IsImageExists")
 	defer span.End()
 
 	_, errQuery := models.Images.Query(ctx, api.db,
-		models.SelectWhere.Images.DeviceID.EQ(device.ID),
+		models.SelectWhere.Images.Device.EQ(device.Slug),
-		models.SelectWhere.Images.PostID.EQ(post.GetID()),
+		models.SelectWhere.Images.PostName.EQ(post.GetName()),
 	).One()
 
 	_, errStat := os.Stat(post.GetImageTargetPath(api.config, device))
@@ -286,6 +310,29 @@ func (api *API) isImageExists(ctx context.Context, post reddit.Post, device *mod
 	return errQuery == nil && errStat == nil
 }
 
+// findImageFileForDevice finds if any of the image file exists for given devices.
+//
+// This helps to avoid downloading the same image for different devices.
+//
+// Return nil if no image file exists for the devices.
+//
+// Ensure to close the file after use.
+func (api *API) findImageFileForDevices(ctx context.Context, post reddit.Post, devices models.DeviceSlice) (file *os.File) {
+	for _, device := range devices {
+		_, err := os.Stat(post.GetImageTargetPath(api.config, device))
+		if err == nil {
+			file, err = os.Open(post.GetImageTargetPath(api.config, device))
+			if err != nil {
+				log.New(ctx).Err(err).Error("failed to open image file", "filename", post.GetImageTargetPath(api.config, device))
+				return nil
+			}
+			return file
+		}
+	}
+
+	return nil
+}
+
 func shouldDownloadPostForDevice(post reddit.Post, device *models.Device) bool {
 	if post.IsNSFW() && device.NSFW == 0 {
 		return false

api/images_list.go

@@ -22,8 +22,8 @@ type ImageListParams struct {
 	Sort      string
 	Offset    int64
 	Limit     int64
-	Device    int32
+	Device    string
-	Subreddit int32
+	Subreddit string
 	CreatedAt time.Time
 }
 
@@ -40,10 +40,8 @@ func (ilp *ImageListParams) FillFromQuery(query Queryable) {
 	if ilp.Limit < 1 {
 		ilp.Limit = 25
 	}
-	device, _ := strconv.ParseInt(query.Get("device"), 10, 32)
+	ilp.Device = query.Get("device")
-	ilp.Device = int32(device)
+	ilp.Subreddit = query.Get("subreddit")
-	subreddit, _ := strconv.ParseInt(query.Get("subreddit"), 10, 32)
-	ilp.Subreddit = int32(subreddit)
 
 	createdAtint, _ := strconv.ParseInt(query.Get("created_at"), 10, 64)
 	if createdAtint > 0 {
@@ -59,8 +57,8 @@ func (ilp ImageListParams) CountQuery() (expr []bob.Mod[*dialect.SelectQuery]) {
 		arg := sqlite.Arg("%" + ilp.Q + "%")
 		expr = append(expr,
 			sm.Where(
-				models.ImageColumns.Title.Like(arg).
+				models.ImageColumns.PostTitle.Like(arg).
-					Or(models.ImageColumns.Poster.Like(arg)).
+					Or(models.ImageColumns.PostAuthor.Like(arg)).
 					Or(models.ImageColumns.ImageRelativePath.Like(arg)),
 			),
 		)
@@ -70,12 +68,12 @@ func (ilp ImageListParams) CountQuery() (expr []bob.Mod[*dialect.SelectQuery]) {
 		expr = append(expr, models.SelectWhere.Images.NSFW.EQ(0))
 	}
 
-	if ilp.Device > 0 {
+	if len(ilp.Device) > 0 {
-		expr = append(expr, models.SelectWhere.Images.DeviceID.EQ(ilp.Device))
+		expr = append(expr, models.SelectWhere.Images.Device.EQ(ilp.Device))
 	}
 
-	if ilp.Subreddit > 0 {
+	if len(ilp.Subreddit) > 0 {
-		expr = append(expr, models.SelectWhere.Images.SubredditID.EQ(ilp.Subreddit))
+		expr = append(expr, models.SelectWhere.Images.Subreddit.EQ(ilp.Subreddit))
 	}
 
 	if !ilp.CreatedAt.IsZero() {

db/migrations/20240406220949_create_subreddits_table.sql

@@ -1,8 +1,7 @@
 -- +goose Up
 -- +goose StatementBegin
 CREATE TABLE subreddits (
-    id INTEGER PRIMARY KEY,
+    name VARCHAR(30) NOT NULL PRIMARY KEY,
-    name VARCHAR(30) NOT NULL,
     enable_schedule INT NOT NULL DEFAULT 1,
     subtype INT NOT NULL DEFAULT 0,
     schedule VARCHAR(20) NOT NULL DEFAULT '@daily',

db/migrations/20240409221254_create_table_device.sql

@@ -1,9 +1,8 @@
 -- +goose Up
 -- +goose StatementBegin
 CREATE TABLE devices(
-    id INTEGER PRIMARY KEY,
+    slug VARCHAR(255) NOT NULL PRIMARY KEY,
     enable INTEGER NOT NULL DEFAULT 1,
-    slug VARCHAR(255) NOT NULL,
     name VARCHAR(255) NOT NULL,
     resolution_x DOUBLE NOT NULL,
     resolution_y DOUBLE NOT NULL,

db/migrations/20240409222145_create_images_table.sql

@@ -2,36 +2,37 @@
 -- +goose StatementBegin
 CREATE TABLE images(
     id INTEGER PRIMARY KEY,
-    subreddit_id INTEGER NOT NULL,
+    subreddit VARCHAR(255) NOT NULL,
-    device_id INTEGER NOT NULL,
+    device VARCHAR(250) NOT NULL,
-    title VARCHAR(255) NOT NULL,
+    post_title VARCHAR(255) NOT NULL,
-    post_id VARCHAR(50) NOT NULL,
+    post_name VARCHAR(255) NOT NULL,
     post_url VARCHAR(255) NOT NULL,
     post_created BIGINT NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    post_name VARCHAR(255) NOT NULL,
+    post_author VARCHAR(50) NOT NULL,
-    poster VARCHAR(50) NOT NULL,
+    post_author_url VARCHAR(255) NOT NULL,
-    poster_url VARCHAR(255) NOT NULL,
     image_relative_path VARCHAR(255) NOT NULL,
-    thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '',
     image_original_url VARCHAR(255) NOT NULL,
-    thumbnail_original_url VARCHAR(255) NOT NULL DEFAULT '',
+    image_height INTEGER NOT NULL DEFAULT 0,
+    image_width INTEGER NOT NULL DEFAULT 0,
+    image_size BIGINT NOT NULL DEFAULT 0,
+    thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '',
     nsfw INTEGER NOT NULL DEFAULT 0,
     created_at BIGINT DEFAULT 0 NOT NULL,
     updated_at BIGINT DEFAULT 0 NOT NULL,
-    CONSTRAINT fk_subreddit_id
+    CONSTRAINT fk_image_subreddit
-        FOREIGN KEY (subreddit_id)
+        FOREIGN KEY (subreddit)
-        REFERENCES subreddits(id)
+        REFERENCES subreddits(name)
         ON DELETE CASCADE,
-    CONSTRAINT fk_devices_id
+    CONSTRAINT fk_image_devices_slug
-        FOREIGN KEY (device_id)
+        FOREIGN KEY (device)
-        REFERENCES devices(id)
+        REFERENCES devices(slug)
         ON DELETE CASCADE
 );
 
-CREATE INDEX idx_subreddit_id_images ON images(subreddit_id);
+CREATE INDEX idx_subreddit_images ON images(subreddit);
 CREATE INDEX idx_nsfw_images ON images(nsfw);
-CREATE INDEX idx_images_created_at ON images(created_at DESC);
+CREATE INDEX idx_images_created_at_nsfw ON images(created_at DESC, nsfw);
-CREATE INDEX idx_unique_device_images_id ON images(device_id, post_id DESC);
+CREATE UNIQUE INDEX idx_unique_images_per_device ON images(device, post_name);
 -- +goose StatementEnd
 
 -- +goose Down

rest/subreddits/check.http

@@ -1,7 +1,6 @@
 POST http://localhost:8080/api/v1/subreddits/check HTTP/1.1
 Host: localhost:8080
 Content-Type: application/json
-Content-Length: 35
 
 {
     "subreddit": "Wallpapers"

server/routes/device_update.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"strconv"
 	"time"
 
 	"github.com/aarondl/opt/omit"
@@ -36,11 +35,10 @@ func (routes *Routes) APIDeviceUpdate(rw http.ResponseWriter, r *http.Request) {
 		dec = json.NewDecoder(r.Body)
 	)
 
-	id, err := strconv.Atoi(chi.URLParam(r, "id"))
+	slug := chi.URLParam(r, "slug")
-	if err != nil {
+	if slug == "" {
-		log.New(ctx).Err(err).Error("failed to parse id")
 		rw.WriteHeader(http.StatusBadRequest)
-		_ = json.NewEncoder(rw).Encode(map[string]string{"error": fmt.Sprintf("bad id: %s", err)})
+		_ = json.NewEncoder(rw).Encode(map[string]string{"error": "missing name"})
 		return
 	}
 
@@ -53,8 +51,7 @@ func (routes *Routes) APIDeviceUpdate(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	device, err := routes.API.DevicesUpdate(ctx, id, &models.DeviceSetter{
+	device, err := routes.API.DevicesUpdate(ctx, slug, &models.DeviceSetter{
-		Slug:                 omit.FromCond(body.Slug, body.Slug != ""),
 		Name:                 omit.FromCond(body.Name, body.Name != ""),
 		ResolutionX:          omit.FromCond(body.ResolutionX, body.ResolutionX != 0),
 		ResolutionY:          omit.FromCond(body.ResolutionY, body.ResolutionY != 0),

server/routes/routes.go

@@ -44,7 +44,7 @@ func (routes *Routes) registerV1APIRoutes(router chi.Router) {
 
 	router.Get("/devices", routes.APIDeviceList)
 	router.Post("/devices", routes.APIDeviceCreate)
-	router.Patch("/devices/{id}", routes.APIDeviceUpdate)
+	router.Patch("/devices/{slug}", routes.APIDeviceUpdate)
 
 	router.Get("/images", routes.ImagesListAPI)
 

views/homeview/homeview.templ

@@ -26,7 +26,7 @@ templ HomeContent(c *views.Context, data Data) {
 			} else {
 				<section class="mb-4 mx-auto">
 					<div
-						class="flex content-center gap-8"
+						class="flex flex-wrap content-center gap-x-8"
 						hx-get="/"
 						hx-target="#recently-added-images"
 						hx-select="#recently-added-images"
@@ -35,7 +35,7 @@ templ HomeContent(c *views.Context, data Data) {
 						hx-include="this"
 						hx-push-url="true"
 					>
-						<h1>
+						<h1 class="mb-4">
 							Recently Added -
 							{ strconv.FormatInt(data.TotalImages, 10) } Images
 						</h1>
@@ -51,7 +51,7 @@ templ HomeContent(c *views.Context, data Data) {
 							<h2 class="mt-4">{ recently.Device.Name }</h2>
 							for _, subreddit := range recently.Subreddits {
 								<h4>
-									<a class="text-primary" href={ templ.SafeURL(fmt.Sprintf("/subreddits/details/%s?device=%d", subreddit.Subreddit.Name, recently.Device.ID)) }>
+									<a class="text-primary" href={ templ.SafeURL(fmt.Sprintf("/subreddits/details/%s?device=%d", subreddit.Subreddit.Name, recently.Device.Slug)) }>
 										{ subreddit.Subreddit.Name }
 									</a>
 									- { strconv.Itoa(len(subreddit.Images)) } images

views/homeview/homeview_data.go

@@ -1,7 +1,6 @@
 package homeview
 
 import (
-	"fmt"
 	"slices"
 	"strings"
 	"time"
@@ -42,11 +41,11 @@ func NewRecentlyAddedImages(images models.ImageSlice) RecentlyAddedImages {
 		}
 		var deviceFound bool
 		for i, ra := range r {
-			if ra.Device.ID == image.R.Device.ID {
+			if ra.Device.Slug == image.R.Device.Slug {
 				deviceFound = true
 				var subredditFound bool
 				for j, subreddit := range r[i].Subreddits {
-					if subreddit.Subreddit.ID == image.R.Subreddit.ID {
+					if subreddit.Subreddit.Name == image.R.Subreddit.Name {
 						subredditFound = true
 						r[i].Subreddits[j].Images = append(r[i].Subreddits[j].Images, image)
 						count++
@@ -89,7 +88,5 @@ func NewRecentlyAddedImages(images models.ImageSlice) RecentlyAddedImages {
 		return strings.Compare(leftName, rightName)
 	})
 
-	fmt.Println("image count", count)
-
 	return r
 }

views/homeview/recently_added_image.templ

@@ -28,7 +28,7 @@ templ RecentlyAddedImageCard(data *models.Image, opts ImageCardOption) {
 				<img
 					class="object-contain w-[256px] h-[256px]"
 					src={ fmt.Sprintf("/img/%s", data.ThumbnailRelativePath) }
-					alt={ data.Title }
+					alt={ data.PostTitle }
 				/>
 			</a>
 		</figure>
@@ -37,9 +37,9 @@ templ RecentlyAddedImageCard(data *models.Image, opts ImageCardOption) {
 				<a
 					href={ templ.URL(data.PostURL) }
 					class="card-title font-bold underline text-base text-primary"
-				>{ truncateTitle(data.Title) }</a>
+				>{ truncateTitle(data.PostTitle) }</a>
 			}
-			<a class="text-primary underline" href={ templ.URL(data.PosterURL) }>{ data.Poster }</a>
+			<a class="text-primary underline" href={ templ.URL(data.PostAuthorURL) }>{ data.PostAuthor }</a>
 			<div>
 				@utils.RelativeTimeNode(fmt.Sprintf("relative-time-%s", data.PostName), data.CreatedAt)
 			</div>