tigor/Redmage
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

refactor: simpler data columns

Browse source
This commit is contained in:
Tigor Hutasuhut 2024-04-30 14:12:33 +07:00
parent 547e73846d
commit 4876930563
13 changed files with 112 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
api/devices_update.go
View file

@ -9,11 +9,11 @@ import (
"github.com/tigorlazuardi/redmage/pkg/errs"
)
func (api *API) DevicesUpdate(ctx context.Context, id int, update *models.DeviceSetter) (device *models.Device, err error) {
func (api *API) DevicesUpdate(ctx context.Context, slug string, update *models.DeviceSetter) (device *models.Device, err error) {
ctx, span := tracer.Start(ctx, "*API.DevicesUpdate")
defer span.End()
device = &models.Device{ID: int32(id)}
device = &models.Device{Slug: slug}
err = models.Devices.Update(ctx, api.db, update, device)
if err != nil {
@ -23,11 +23,11 @@ func (api *API) DevicesUpdate(ctx context.Context, id int, update *models.Device
return device, errs.Wrapw(err, "a device with the same slug id already exists").Code(409)
}
}
return device, errs.Wrapw(err, "failed to update device", "id", id, "values", update)
return device, errs.Wrapw(err, "failed to update device", "slug", slug, "values", update)
}
if err := device.Reload(ctx, api.db); err != nil {
return device, errs.Wrapw(err, "failed to reload device", "id", id)
return device, errs.Wrapw(err, "failed to reload device", "slug", slug)
}
return device, nil

81
api/download_subreddit_images.go
View file

@ -100,11 +100,11 @@ func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.List
if !post.IsImagePost() {
continue
}
devices := api.getDevicesThatAcceptPost(ctx, post, devices)
if len(devices) == 0 {
acceptedDevices := api.getDevicesThatAcceptPost(ctx, post, devices)
if len(acceptedDevices) == 0 {
continue
}
log.New(ctx).Debug("downloading image", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", devices)
log.New(ctx).Debug("downloading image", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", acceptedDevices)
wg.Add(1)
api.imageSemaphore <- struct{}{}
go func(ctx context.Context, post reddit.Post) {
@ -113,7 +113,15 @@ func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.List
wg.Done()
}()
if err := api.downloadSubredditImage(ctx, post, subreddit, devices); err != nil {
if imageFile := api.findImageFileForDevices(ctx, post, devices); imageFile != nil {
err := api.saveImageToFSAndDatabase(ctx, imageFile, subreddit, post, acceptedDevices)
if err != nil {
log.New(ctx).Err(err).Error("failed to download subreddit image")
}
return
}
if err := api.downloadSubredditImage(ctx, post, subreddit, acceptedDevices); err != nil {
log.New(ctx).Err(err).Error("failed to download subreddit image")
}
}(ctx, post)
@ -174,13 +182,21 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su
return errs.Wrapw(err, "failed to encode thumbnail file to jpeg", "filename", thumbnailPath)
}
return api.saveImageToFSAndDatabase(ctx, tmpImageFile, subreddit, post, devices)
}
func (api *API) saveImageToFSAndDatabase(ctx context.Context, image io.ReadCloser, subreddit *models.Subreddit, post reddit.Post, devices models.DeviceSlice) (err error) {
ctx, span := tracer.Start(ctx, "*API.saveImageToFSAndDatabase")
defer span.End()
defer image.Close()
w, close, err := api.createDeviceImageWriters(post, devices)
if err != nil {
return errs.Wrapw(err, "failed to create image files")
}
log.New(ctx).Debug("saving image files", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", devices)
defer close()
_, err = io.Copy(w, tmpImageFile)
_, err = io.Copy(w, image)
if err != nil {
return errs.Wrapw(err, "failed to save image files")
}
@ -192,20 +208,27 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su
if post.IsNSFW() {
nsfw = 1
}
width, height := post.GetImageSize()
var size int64
if fi, err := os.Stat(post.GetImageTargetPath(api.config, device)); err == nil {
size = fi.Size()
}
many = append(many, &models.ImageSetter{
SubredditID: omit.From(subreddit.ID),
DeviceID: omit.From(device.ID),
Title: omit.From(post.GetTitle()),
PostID: omit.From(post.GetID()),
Subreddit: omit.From(subreddit.Name),
Device: omit.From(device.Slug),
PostTitle: omit.From(post.GetTitle()),
PostURL: omit.From(post.GetPostURL()),
PostCreated: omit.From(post.GetCreated().Unix()),
PostName: omit.From(post.GetName()),
Poster: omit.From(post.GetAuthor()),
PosterURL: omit.From(post.GetAuthorURL()),
PostAuthor: omit.From(post.GetAuthor()),
PostAuthorURL: omit.From(post.GetAuthorURL()),
ImageWidth: omit.From(int32(width)),
ImageHeight: omit.From(int32(height)),
ImageSize: omit.From(size),
ImageRelativePath: omit.From(post.GetImageRelativePath(device)),
ThumbnailRelativePath: omit.From(post.GetThumbnailRelativePath()),
ImageOriginalURL: omit.From(post.GetImageURL()),
ThumbnailOriginalURL: omit.From(post.GetThumbnailURL()),
NSFW: omit.From(nsfw),
CreatedAt: omit.From(now.Unix()),
UpdatedAt: omit.From(now.Unix()),
@ -217,7 +240,6 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su
if err != nil {
return errs.Wrapw(err, "failed to insert images to database", "params", many)
}
return nil
}
@ -265,20 +287,22 @@ func (api *API) createDeviceImageWriters(post reddit.Post, devices models.Device
func (api *API) getDevicesThatAcceptPost(ctx context.Context, post reddit.Post, devices models.DeviceSlice) (devs models.DeviceSlice) {
for _, device := range devices {
if shouldDownloadPostForDevice(post, device) && !api.isImageExists(ctx, post, device) {
if shouldDownloadPostForDevice(post, device) && !api.isImageEntryExists(ctx, post, device) {
devs = append(devs, device)
}
}
return devs
}
func (api *API) isImageExists(ctx context.Context, post reddit.Post, device *models.Device) (found bool) {
// isImageEntryExists checks if the image entry already exists in the database and
// the image file actually exists in the filesystem.
func (api *API) isImageEntryExists(ctx context.Context, post reddit.Post, device *models.Device) (found bool) {
ctx, span := tracer.Start(ctx, "*API.IsImageExists")
defer span.End()
_, errQuery := models.Images.Query(ctx, api.db,
models.SelectWhere.Images.DeviceID.EQ(device.ID),
models.SelectWhere.Images.PostID.EQ(post.GetID()),
models.SelectWhere.Images.Device.EQ(device.Slug),
models.SelectWhere.Images.PostName.EQ(post.GetName()),
).One()
_, errStat := os.Stat(post.GetImageTargetPath(api.config, device))
@ -286,6 +310,29 @@ func (api *API) isImageExists(ctx context.Context, post reddit.Post, device *mod
return errQuery == nil && errStat == nil
}
// findImageFileForDevice finds if any of the image file exists for given devices.
//
// This helps to avoid downloading the same image for different devices.
//
// Return nil if no image file exists for the devices.
//
// Ensure to close the file after use.
func (api *API) findImageFileForDevices(ctx context.Context, post reddit.Post, devices models.DeviceSlice) (file *os.File) {
for _, device := range devices {
_, err := os.Stat(post.GetImageTargetPath(api.config, device))
if err == nil {
file, err = os.Open(post.GetImageTargetPath(api.config, device))
if err != nil {
log.New(ctx).Err(err).Error("failed to open image file", "filename", post.GetImageTargetPath(api.config, device))
return nil
}
return file
}
}
return nil
}
func shouldDownloadPostForDevice(post reddit.Post, device *models.Device) bool {
if post.IsNSFW() && device.NSFW == 0 {
return false

22
api/images_list.go
View file

@ -22,8 +22,8 @@ type ImageListParams struct {
Sort string
Offset int64
Limit int64
Device int32
Subreddit int32
Device string
Subreddit string
CreatedAt time.Time
}
@ -40,10 +40,8 @@ func (ilp *ImageListParams) FillFromQuery(query Queryable) {
if ilp.Limit < 1 {
ilp.Limit = 25
}
device, _ := strconv.ParseInt(query.Get("device"), 10, 32)
ilp.Device = int32(device)
subreddit, _ := strconv.ParseInt(query.Get("subreddit"), 10, 32)
ilp.Subreddit = int32(subreddit)
ilp.Device = query.Get("device")
ilp.Subreddit = query.Get("subreddit")
createdAtint, _ := strconv.ParseInt(query.Get("created_at"), 10, 64)
if createdAtint > 0 {
@ -59,8 +57,8 @@ func (ilp ImageListParams) CountQuery() (expr []bob.Mod[*dialect.SelectQuery]) {
arg := sqlite.Arg("%" + ilp.Q + "%")
expr = append(expr,
sm.Where(
models.ImageColumns.Title.Like(arg).
Or(models.ImageColumns.Poster.Like(arg)).
models.ImageColumns.PostTitle.Like(arg).
Or(models.ImageColumns.PostAuthor.Like(arg)).
Or(models.ImageColumns.ImageRelativePath.Like(arg)),
),
)
@ -70,12 +68,12 @@ func (ilp ImageListParams) CountQuery() (expr []bob.Mod[*dialect.SelectQuery]) {
expr = append(expr, models.SelectWhere.Images.NSFW.EQ(0))
}
if ilp.Device > 0 {
expr = append(expr, models.SelectWhere.Images.DeviceID.EQ(ilp.Device))
if len(ilp.Device) > 0 {
expr = append(expr, models.SelectWhere.Images.Device.EQ(ilp.Device))
}
if ilp.Subreddit > 0 {
expr = append(expr, models.SelectWhere.Images.SubredditID.EQ(ilp.Subreddit))
if len(ilp.Subreddit) > 0 {
expr = append(expr, models.SelectWhere.Images.Subreddit.EQ(ilp.Subreddit))
}
if !ilp.CreatedAt.IsZero() {

3
db/migrations/20240406220949_create_subreddits_table.sql
View file

@ -1,8 +1,7 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE subreddits (
id INTEGER PRIMARY KEY,
name VARCHAR(30) NOT NULL,
name VARCHAR(30) NOT NULL PRIMARY KEY,
enable_schedule INT NOT NULL DEFAULT 1,
subtype INT NOT NULL DEFAULT 0,
schedule VARCHAR(20) NOT NULL DEFAULT '@daily',

3
db/migrations/20240409221254_create_table_device.sql
View file

@ -1,9 +1,8 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE devices(
id INTEGER PRIMARY KEY,
slug VARCHAR(255) NOT NULL PRIMARY KEY,
enable INTEGER NOT NULL DEFAULT 1,
slug VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL,
resolution_x DOUBLE NOT NULL,
resolution_y DOUBLE NOT NULL,

37
db/migrations/20240409222145_create_images_table.sql
View file

@ -2,36 +2,37 @@
-- +goose StatementBegin
CREATE TABLE images(
id INTEGER PRIMARY KEY,
subreddit_id INTEGER NOT NULL,
device_id INTEGER NOT NULL,
title VARCHAR(255) NOT NULL,
post_id VARCHAR(50) NOT NULL,
subreddit VARCHAR(255) NOT NULL,
device VARCHAR(250) NOT NULL,
post_title VARCHAR(255) NOT NULL,
post_name VARCHAR(255) NOT NULL,
post_url VARCHAR(255) NOT NULL,
post_created BIGINT NOT NULL DEFAULT CURRENT_TIMESTAMP,
post_name VARCHAR(255) NOT NULL,
poster VARCHAR(50) NOT NULL,
poster_url VARCHAR(255) NOT NULL,
post_author VARCHAR(50) NOT NULL,
post_author_url VARCHAR(255) NOT NULL,
image_relative_path VARCHAR(255) NOT NULL,
thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '',
image_original_url VARCHAR(255) NOT NULL,
thumbnail_original_url VARCHAR(255) NOT NULL DEFAULT '',
image_height INTEGER NOT NULL DEFAULT 0,
image_width INTEGER NOT NULL DEFAULT 0,
image_size BIGINT NOT NULL DEFAULT 0,
thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '',
nsfw INTEGER NOT NULL DEFAULT 0,
created_at BIGINT DEFAULT 0 NOT NULL,
updated_at BIGINT DEFAULT 0 NOT NULL,
CONSTRAINT fk_subreddit_id
FOREIGN KEY (subreddit_id)
REFERENCES subreddits(id)
CONSTRAINT fk_image_subreddit
FOREIGN KEY (subreddit)
REFERENCES subreddits(name)
ON DELETE CASCADE,
CONSTRAINT fk_devices_id
FOREIGN KEY (device_id)
REFERENCES devices(id)
CONSTRAINT fk_image_devices_slug
FOREIGN KEY (device)
REFERENCES devices(slug)
ON DELETE CASCADE
);
CREATE INDEX idx_subreddit_id_images ON images(subreddit_id);
CREATE INDEX idx_subreddit_images ON images(subreddit);
CREATE INDEX idx_nsfw_images ON images(nsfw);
CREATE INDEX idx_images_created_at ON images(created_at DESC);
CREATE INDEX idx_unique_device_images_id ON images(device_id, post_id DESC);
CREATE INDEX idx_images_created_at_nsfw ON images(created_at DESC, nsfw);
CREATE UNIQUE INDEX idx_unique_images_per_device ON images(device, post_name);
-- +goose StatementEnd
-- +goose Down

BIN
pubsub.db-journal Normal file
View file

Binary file not shown.

1
rest/subreddits/check.http
View file

@ -1,7 +1,6 @@
POST http://localhost:8080/api/v1/subreddits/check HTTP/1.1
Host: localhost:8080
Content-Type: application/json
Content-Length: 35
{
"subreddit": "Wallpapers"

11
server/routes/device_update.go
View file

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"time"
"github.com/aarondl/opt/omit"
@ -36,11 +35,10 @@ func (routes *Routes) APIDeviceUpdate(rw http.ResponseWriter, r *http.Request) {
dec = json.NewDecoder(r.Body)
)
id, err := strconv.Atoi(chi.URLParam(r, "id"))
if err != nil {
log.New(ctx).Err(err).Error("failed to parse id")
slug := chi.URLParam(r, "slug")
if slug == "" {
rw.WriteHeader(http.StatusBadRequest)
_ = json.NewEncoder(rw).Encode(map[string]string{"error": fmt.Sprintf("bad id: %s", err)})
_ = json.NewEncoder(rw).Encode(map[string]string{"error": "missing name"})
return
}
@ -53,8 +51,7 @@ func (routes *Routes) APIDeviceUpdate(rw http.ResponseWriter, r *http.Request) {
return
}
device, err := routes.API.DevicesUpdate(ctx, id, &models.DeviceSetter{
Slug: omit.FromCond(body.Slug, body.Slug != ""),
device, err := routes.API.DevicesUpdate(ctx, slug, &models.DeviceSetter{
Name: omit.FromCond(body.Name, body.Name != ""),
ResolutionX: omit.FromCond(body.ResolutionX, body.ResolutionX != 0),
ResolutionY: omit.FromCond(body.ResolutionY, body.ResolutionY != 0),

2
server/routes/routes.go
View file

@ -44,7 +44,7 @@ func (routes *Routes) registerV1APIRoutes(router chi.Router) {
router.Get("/devices", routes.APIDeviceList)
router.Post("/devices", routes.APIDeviceCreate)
router.Patch("/devices/{id}", routes.APIDeviceUpdate)
router.Patch("/devices/{slug}", routes.APIDeviceUpdate)
router.Get("/images", routes.ImagesListAPI)

6
views/homeview/homeview.templ
View file

@ -26,7 +26,7 @@ templ HomeContent(c *views.Context, data Data) {
} else {
<section class="mb-4 mx-auto">
<div
class="flex content-center gap-8"
class="flex flex-wrap content-center gap-x-8"
hx-get="/"
hx-target="#recently-added-images"
hx-select="#recently-added-images"
@ -35,7 +35,7 @@ templ HomeContent(c *views.Context, data Data) {
hx-include="this"
hx-push-url="true"
>
<h1>
<h1 class="mb-4">
Recently Added -
{ strconv.FormatInt(data.TotalImages, 10) } Images
</h1>
@ -51,7 +51,7 @@ templ HomeContent(c *views.Context, data Data) {
<h2 class="mt-4">{ recently.Device.Name }</h2>
for _, subreddit := range recently.Subreddits {
<h4>
<a class="text-primary" href={ templ.SafeURL(fmt.Sprintf("/subreddits/details/%s?device=%d", subreddit.Subreddit.Name, recently.Device.ID)) }>
<a class="text-primary" href={ templ.SafeURL(fmt.Sprintf("/subreddits/details/%s?device=%d", subreddit.Subreddit.Name, recently.Device.Slug)) }>
{ subreddit.Subreddit.Name }
</a>
- { strconv.Itoa(len(subreddit.Images)) } images

7
views/homeview/homeview_data.go
View file

@ -1,7 +1,6 @@
package homeview
import (
"fmt"
"slices"
"strings"
"time"
@ -42,11 +41,11 @@ func NewRecentlyAddedImages(images models.ImageSlice) RecentlyAddedImages {
}
var deviceFound bool
for i, ra := range r {
if ra.Device.ID == image.R.Device.ID {
if ra.Device.Slug == image.R.Device.Slug {
deviceFound = true
var subredditFound bool
for j, subreddit := range r[i].Subreddits {
if subreddit.Subreddit.ID == image.R.Subreddit.ID {
if subreddit.Subreddit.Name == image.R.Subreddit.Name {
subredditFound = true
r[i].Subreddits[j].Images = append(r[i].Subreddits[j].Images, image)
count++
@ -89,7 +88,5 @@ func NewRecentlyAddedImages(images models.ImageSlice) RecentlyAddedImages {
return strings.Compare(leftName, rightName)
})
fmt.Println("image count", count)
return r
}

6
views/homeview/recently_added_image.templ
View file

@ -28,7 +28,7 @@ templ RecentlyAddedImageCard(data *models.Image, opts ImageCardOption) {
<img
class="object-contain w-[256px] h-[256px]"
src={ fmt.Sprintf("/img/%s", data.ThumbnailRelativePath) }
alt={ data.Title }
alt={ data.PostTitle }
/>
</a>
</figure>
@ -37,9 +37,9 @@ templ RecentlyAddedImageCard(data *models.Image, opts ImageCardOption) {
<a
href={ templ.URL(data.PostURL) }
class="card-title font-bold underline text-base text-primary"
>{ truncateTitle(data.Title) }</a>
>{ truncateTitle(data.PostTitle) }</a>
}
<a class="text-primary underline" href={ templ.URL(data.PosterURL) }>{ data.Poster }</a>
<a class="text-primary underline" href={ templ.URL(data.PostAuthorURL) }>{ data.PostAuthor }</a>
<div>
@utils.RelativeTimeNode(fmt.Sprintf("relative-time-%s", data.PostName), data.CreatedAt)
</div>