From 487693056375abf71ae3bbfe3f0ef68c382b07b2 Mon Sep 17 00:00:00 2001 From: Tigor Hutasuhut Date: Tue, 30 Apr 2024 14:12:33 +0700 Subject: [PATCH] refactor: simpler data columns --- api/devices_update.go | 8 +- api/download_subreddit_images.go | 81 ++++++++++++++---- api/images_list.go | 22 +++-- ...20240406220949_create_subreddits_table.sql | 3 +- .../20240409221254_create_table_device.sql | 3 +- .../20240409222145_create_images_table.sql | 37 ++++---- pubsub.db-journal | Bin 0 -> 4616 bytes rest/subreddits/check.http | 1 - server/routes/device_update.go | 11 +-- server/routes/routes.go | 2 +- views/homeview/homeview.templ | 6 +- views/homeview/homeview_data.go | 7 +- views/homeview/recently_added_image.templ | 6 +- 13 files changed, 112 insertions(+), 75 deletions(-) create mode 100644 pubsub.db-journal diff --git a/api/devices_update.go b/api/devices_update.go index 4fa04b3..2291224 100644 --- a/api/devices_update.go +++ b/api/devices_update.go @@ -9,11 +9,11 @@ import ( "github.com/tigorlazuardi/redmage/pkg/errs" ) -func (api *API) DevicesUpdate(ctx context.Context, id int, update *models.DeviceSetter) (device *models.Device, err error) { +func (api *API) DevicesUpdate(ctx context.Context, slug string, update *models.DeviceSetter) (device *models.Device, err error) { ctx, span := tracer.Start(ctx, "*API.DevicesUpdate") defer span.End() - device = &models.Device{ID: int32(id)} + device = &models.Device{Slug: slug} err = models.Devices.Update(ctx, api.db, update, device) if err != nil { @@ -23,11 +23,11 @@ func (api *API) DevicesUpdate(ctx context.Context, id int, update *models.Device return device, errs.Wrapw(err, "a device with the same slug id already exists").Code(409) } } - return device, errs.Wrapw(err, "failed to update device", "id", id, "values", update) + return device, errs.Wrapw(err, "failed to update device", "slug", slug, "values", update) } if err := device.Reload(ctx, api.db); err != nil { - return device, errs.Wrapw(err, "failed to reload device", "id", id) + return device, errs.Wrapw(err, "failed to reload device", "slug", slug) } return device, nil diff --git a/api/download_subreddit_images.go b/api/download_subreddit_images.go index d5c22d5..955e04a 100644 --- a/api/download_subreddit_images.go +++ b/api/download_subreddit_images.go @@ -100,11 +100,11 @@ func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.List if !post.IsImagePost() { continue } - devices := api.getDevicesThatAcceptPost(ctx, post, devices) - if len(devices) == 0 { + acceptedDevices := api.getDevicesThatAcceptPost(ctx, post, devices) + if len(acceptedDevices) == 0 { continue } - log.New(ctx).Debug("downloading image", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", devices) + log.New(ctx).Debug("downloading image", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", acceptedDevices) wg.Add(1) api.imageSemaphore <- struct{}{} go func(ctx context.Context, post reddit.Post) { @@ -113,7 +113,15 @@ func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.List wg.Done() }() - if err := api.downloadSubredditImage(ctx, post, subreddit, devices); err != nil { + if imageFile := api.findImageFileForDevices(ctx, post, devices); imageFile != nil { + err := api.saveImageToFSAndDatabase(ctx, imageFile, subreddit, post, acceptedDevices) + if err != nil { + log.New(ctx).Err(err).Error("failed to download subreddit image") + } + return + } + + if err := api.downloadSubredditImage(ctx, post, subreddit, acceptedDevices); err != nil { log.New(ctx).Err(err).Error("failed to download subreddit image") } }(ctx, post) @@ -174,13 +182,21 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su return errs.Wrapw(err, "failed to encode thumbnail file to jpeg", "filename", thumbnailPath) } + return api.saveImageToFSAndDatabase(ctx, tmpImageFile, subreddit, post, devices) +} + +func (api *API) saveImageToFSAndDatabase(ctx context.Context, image io.ReadCloser, subreddit *models.Subreddit, post reddit.Post, devices models.DeviceSlice) (err error) { + ctx, span := tracer.Start(ctx, "*API.saveImageToFSAndDatabase") + defer span.End() + defer image.Close() + w, close, err := api.createDeviceImageWriters(post, devices) if err != nil { return errs.Wrapw(err, "failed to create image files") } log.New(ctx).Debug("saving image files", "post_id", post.GetID(), "post_url", post.GetImageURL(), "devices", devices) defer close() - _, err = io.Copy(w, tmpImageFile) + _, err = io.Copy(w, image) if err != nil { return errs.Wrapw(err, "failed to save image files") } @@ -192,20 +208,27 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su if post.IsNSFW() { nsfw = 1 } + width, height := post.GetImageSize() + var size int64 + if fi, err := os.Stat(post.GetImageTargetPath(api.config, device)); err == nil { + size = fi.Size() + } + many = append(many, &models.ImageSetter{ - SubredditID: omit.From(subreddit.ID), - DeviceID: omit.From(device.ID), - Title: omit.From(post.GetTitle()), - PostID: omit.From(post.GetID()), + Subreddit: omit.From(subreddit.Name), + Device: omit.From(device.Slug), + PostTitle: omit.From(post.GetTitle()), PostURL: omit.From(post.GetPostURL()), PostCreated: omit.From(post.GetCreated().Unix()), PostName: omit.From(post.GetName()), - Poster: omit.From(post.GetAuthor()), - PosterURL: omit.From(post.GetAuthorURL()), + PostAuthor: omit.From(post.GetAuthor()), + PostAuthorURL: omit.From(post.GetAuthorURL()), + ImageWidth: omit.From(int32(width)), + ImageHeight: omit.From(int32(height)), + ImageSize: omit.From(size), ImageRelativePath: omit.From(post.GetImageRelativePath(device)), ThumbnailRelativePath: omit.From(post.GetThumbnailRelativePath()), ImageOriginalURL: omit.From(post.GetImageURL()), - ThumbnailOriginalURL: omit.From(post.GetThumbnailURL()), NSFW: omit.From(nsfw), CreatedAt: omit.From(now.Unix()), UpdatedAt: omit.From(now.Unix()), @@ -217,7 +240,6 @@ func (api *API) downloadSubredditImage(ctx context.Context, post reddit.Post, su if err != nil { return errs.Wrapw(err, "failed to insert images to database", "params", many) } - return nil } @@ -265,20 +287,22 @@ func (api *API) createDeviceImageWriters(post reddit.Post, devices models.Device func (api *API) getDevicesThatAcceptPost(ctx context.Context, post reddit.Post, devices models.DeviceSlice) (devs models.DeviceSlice) { for _, device := range devices { - if shouldDownloadPostForDevice(post, device) && !api.isImageExists(ctx, post, device) { + if shouldDownloadPostForDevice(post, device) && !api.isImageEntryExists(ctx, post, device) { devs = append(devs, device) } } return devs } -func (api *API) isImageExists(ctx context.Context, post reddit.Post, device *models.Device) (found bool) { +// isImageEntryExists checks if the image entry already exists in the database and +// the image file actually exists in the filesystem. +func (api *API) isImageEntryExists(ctx context.Context, post reddit.Post, device *models.Device) (found bool) { ctx, span := tracer.Start(ctx, "*API.IsImageExists") defer span.End() _, errQuery := models.Images.Query(ctx, api.db, - models.SelectWhere.Images.DeviceID.EQ(device.ID), - models.SelectWhere.Images.PostID.EQ(post.GetID()), + models.SelectWhere.Images.Device.EQ(device.Slug), + models.SelectWhere.Images.PostName.EQ(post.GetName()), ).One() _, errStat := os.Stat(post.GetImageTargetPath(api.config, device)) @@ -286,6 +310,29 @@ func (api *API) isImageExists(ctx context.Context, post reddit.Post, device *mod return errQuery == nil && errStat == nil } +// findImageFileForDevice finds if any of the image file exists for given devices. +// +// This helps to avoid downloading the same image for different devices. +// +// Return nil if no image file exists for the devices. +// +// Ensure to close the file after use. +func (api *API) findImageFileForDevices(ctx context.Context, post reddit.Post, devices models.DeviceSlice) (file *os.File) { + for _, device := range devices { + _, err := os.Stat(post.GetImageTargetPath(api.config, device)) + if err == nil { + file, err = os.Open(post.GetImageTargetPath(api.config, device)) + if err != nil { + log.New(ctx).Err(err).Error("failed to open image file", "filename", post.GetImageTargetPath(api.config, device)) + return nil + } + return file + } + } + + return nil +} + func shouldDownloadPostForDevice(post reddit.Post, device *models.Device) bool { if post.IsNSFW() && device.NSFW == 0 { return false diff --git a/api/images_list.go b/api/images_list.go index 7cee855..135554c 100644 --- a/api/images_list.go +++ b/api/images_list.go @@ -22,8 +22,8 @@ type ImageListParams struct { Sort string Offset int64 Limit int64 - Device int32 - Subreddit int32 + Device string + Subreddit string CreatedAt time.Time } @@ -40,10 +40,8 @@ func (ilp *ImageListParams) FillFromQuery(query Queryable) { if ilp.Limit < 1 { ilp.Limit = 25 } - device, _ := strconv.ParseInt(query.Get("device"), 10, 32) - ilp.Device = int32(device) - subreddit, _ := strconv.ParseInt(query.Get("subreddit"), 10, 32) - ilp.Subreddit = int32(subreddit) + ilp.Device = query.Get("device") + ilp.Subreddit = query.Get("subreddit") createdAtint, _ := strconv.ParseInt(query.Get("created_at"), 10, 64) if createdAtint > 0 { @@ -59,8 +57,8 @@ func (ilp ImageListParams) CountQuery() (expr []bob.Mod[*dialect.SelectQuery]) { arg := sqlite.Arg("%" + ilp.Q + "%") expr = append(expr, sm.Where( - models.ImageColumns.Title.Like(arg). - Or(models.ImageColumns.Poster.Like(arg)). + models.ImageColumns.PostTitle.Like(arg). + Or(models.ImageColumns.PostAuthor.Like(arg)). Or(models.ImageColumns.ImageRelativePath.Like(arg)), ), ) @@ -70,12 +68,12 @@ func (ilp ImageListParams) CountQuery() (expr []bob.Mod[*dialect.SelectQuery]) { expr = append(expr, models.SelectWhere.Images.NSFW.EQ(0)) } - if ilp.Device > 0 { - expr = append(expr, models.SelectWhere.Images.DeviceID.EQ(ilp.Device)) + if len(ilp.Device) > 0 { + expr = append(expr, models.SelectWhere.Images.Device.EQ(ilp.Device)) } - if ilp.Subreddit > 0 { - expr = append(expr, models.SelectWhere.Images.SubredditID.EQ(ilp.Subreddit)) + if len(ilp.Subreddit) > 0 { + expr = append(expr, models.SelectWhere.Images.Subreddit.EQ(ilp.Subreddit)) } if !ilp.CreatedAt.IsZero() { diff --git a/db/migrations/20240406220949_create_subreddits_table.sql b/db/migrations/20240406220949_create_subreddits_table.sql index e751d59..c914bd9 100644 --- a/db/migrations/20240406220949_create_subreddits_table.sql +++ b/db/migrations/20240406220949_create_subreddits_table.sql @@ -1,8 +1,7 @@ -- +goose Up -- +goose StatementBegin CREATE TABLE subreddits ( - id INTEGER PRIMARY KEY, - name VARCHAR(30) NOT NULL, + name VARCHAR(30) NOT NULL PRIMARY KEY, enable_schedule INT NOT NULL DEFAULT 1, subtype INT NOT NULL DEFAULT 0, schedule VARCHAR(20) NOT NULL DEFAULT '@daily', diff --git a/db/migrations/20240409221254_create_table_device.sql b/db/migrations/20240409221254_create_table_device.sql index bea6789..40069cb 100644 --- a/db/migrations/20240409221254_create_table_device.sql +++ b/db/migrations/20240409221254_create_table_device.sql @@ -1,9 +1,8 @@ -- +goose Up -- +goose StatementBegin CREATE TABLE devices( - id INTEGER PRIMARY KEY, + slug VARCHAR(255) NOT NULL PRIMARY KEY, enable INTEGER NOT NULL DEFAULT 1, - slug VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL, resolution_x DOUBLE NOT NULL, resolution_y DOUBLE NOT NULL, diff --git a/db/migrations/20240409222145_create_images_table.sql b/db/migrations/20240409222145_create_images_table.sql index 12a1046..3cebedf 100644 --- a/db/migrations/20240409222145_create_images_table.sql +++ b/db/migrations/20240409222145_create_images_table.sql @@ -2,36 +2,37 @@ -- +goose StatementBegin CREATE TABLE images( id INTEGER PRIMARY KEY, - subreddit_id INTEGER NOT NULL, - device_id INTEGER NOT NULL, - title VARCHAR(255) NOT NULL, - post_id VARCHAR(50) NOT NULL, + subreddit VARCHAR(255) NOT NULL, + device VARCHAR(250) NOT NULL, + post_title VARCHAR(255) NOT NULL, + post_name VARCHAR(255) NOT NULL, post_url VARCHAR(255) NOT NULL, post_created BIGINT NOT NULL DEFAULT CURRENT_TIMESTAMP, - post_name VARCHAR(255) NOT NULL, - poster VARCHAR(50) NOT NULL, - poster_url VARCHAR(255) NOT NULL, + post_author VARCHAR(50) NOT NULL, + post_author_url VARCHAR(255) NOT NULL, image_relative_path VARCHAR(255) NOT NULL, - thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '', image_original_url VARCHAR(255) NOT NULL, - thumbnail_original_url VARCHAR(255) NOT NULL DEFAULT '', + image_height INTEGER NOT NULL DEFAULT 0, + image_width INTEGER NOT NULL DEFAULT 0, + image_size BIGINT NOT NULL DEFAULT 0, + thumbnail_relative_path VARCHAR(255) NOT NULL DEFAULT '', nsfw INTEGER NOT NULL DEFAULT 0, created_at BIGINT DEFAULT 0 NOT NULL, updated_at BIGINT DEFAULT 0 NOT NULL, - CONSTRAINT fk_subreddit_id - FOREIGN KEY (subreddit_id) - REFERENCES subreddits(id) + CONSTRAINT fk_image_subreddit + FOREIGN KEY (subreddit) + REFERENCES subreddits(name) ON DELETE CASCADE, - CONSTRAINT fk_devices_id - FOREIGN KEY (device_id) - REFERENCES devices(id) + CONSTRAINT fk_image_devices_slug + FOREIGN KEY (device) + REFERENCES devices(slug) ON DELETE CASCADE ); -CREATE INDEX idx_subreddit_id_images ON images(subreddit_id); +CREATE INDEX idx_subreddit_images ON images(subreddit); CREATE INDEX idx_nsfw_images ON images(nsfw); -CREATE INDEX idx_images_created_at ON images(created_at DESC); -CREATE INDEX idx_unique_device_images_id ON images(device_id, post_id DESC); +CREATE INDEX idx_images_created_at_nsfw ON images(created_at DESC, nsfw); +CREATE UNIQUE INDEX idx_unique_images_per_device ON images(device, post_name); -- +goose StatementEnd -- +goose Down diff --git a/pubsub.db-journal b/pubsub.db-journal new file mode 100644 index 0000000000000000000000000000000000000000..28d7df7f268c15a1be877cd40c3a0649f7b500c3 GIT binary patch literal 4616 zcmZQzK!Cb=mnQ>xtPBiHKuQ21G0GawA;7{5^c^GrM+P7o1*0J_8UmvsFd71*Aut*O tqaiRF0;3@?8UmvsFd71*Aut*O!zBcG8Cj$m8H-X=aud^2nV3N1J^%<=3N`=$ literal 0 HcmV?d00001 diff --git a/rest/subreddits/check.http b/rest/subreddits/check.http index acb7c1d..81231fc 100644 --- a/rest/subreddits/check.http +++ b/rest/subreddits/check.http @@ -1,7 +1,6 @@ POST http://localhost:8080/api/v1/subreddits/check HTTP/1.1 Host: localhost:8080 Content-Type: application/json -Content-Length: 35 { "subreddit": "Wallpapers" diff --git a/server/routes/device_update.go b/server/routes/device_update.go index ad05fbb..b222a61 100644 --- a/server/routes/device_update.go +++ b/server/routes/device_update.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "net/http" - "strconv" "time" "github.com/aarondl/opt/omit" @@ -36,11 +35,10 @@ func (routes *Routes) APIDeviceUpdate(rw http.ResponseWriter, r *http.Request) { dec = json.NewDecoder(r.Body) ) - id, err := strconv.Atoi(chi.URLParam(r, "id")) - if err != nil { - log.New(ctx).Err(err).Error("failed to parse id") + slug := chi.URLParam(r, "slug") + if slug == "" { rw.WriteHeader(http.StatusBadRequest) - _ = json.NewEncoder(rw).Encode(map[string]string{"error": fmt.Sprintf("bad id: %s", err)}) + _ = json.NewEncoder(rw).Encode(map[string]string{"error": "missing name"}) return } @@ -53,8 +51,7 @@ func (routes *Routes) APIDeviceUpdate(rw http.ResponseWriter, r *http.Request) { return } - device, err := routes.API.DevicesUpdate(ctx, id, &models.DeviceSetter{ - Slug: omit.FromCond(body.Slug, body.Slug != ""), + device, err := routes.API.DevicesUpdate(ctx, slug, &models.DeviceSetter{ Name: omit.FromCond(body.Name, body.Name != ""), ResolutionX: omit.FromCond(body.ResolutionX, body.ResolutionX != 0), ResolutionY: omit.FromCond(body.ResolutionY, body.ResolutionY != 0), diff --git a/server/routes/routes.go b/server/routes/routes.go index b53b8e5..4b3de19 100644 --- a/server/routes/routes.go +++ b/server/routes/routes.go @@ -44,7 +44,7 @@ func (routes *Routes) registerV1APIRoutes(router chi.Router) { router.Get("/devices", routes.APIDeviceList) router.Post("/devices", routes.APIDeviceCreate) - router.Patch("/devices/{id}", routes.APIDeviceUpdate) + router.Patch("/devices/{slug}", routes.APIDeviceUpdate) router.Get("/images", routes.ImagesListAPI) diff --git a/views/homeview/homeview.templ b/views/homeview/homeview.templ index 00b3f20..2d4ee59 100644 --- a/views/homeview/homeview.templ +++ b/views/homeview/homeview.templ @@ -26,7 +26,7 @@ templ HomeContent(c *views.Context, data Data) { } else {
-

+

Recently Added - { strconv.FormatInt(data.TotalImages, 10) } Images

@@ -51,7 +51,7 @@ templ HomeContent(c *views.Context, data Data) {

{ recently.Device.Name }

for _, subreddit := range recently.Subreddits {

- + { subreddit.Subreddit.Name } - { strconv.Itoa(len(subreddit.Images)) } images diff --git a/views/homeview/homeview_data.go b/views/homeview/homeview_data.go index 3feef56..feaf7fe 100644 --- a/views/homeview/homeview_data.go +++ b/views/homeview/homeview_data.go @@ -1,7 +1,6 @@ package homeview import ( - "fmt" "slices" "strings" "time" @@ -42,11 +41,11 @@ func NewRecentlyAddedImages(images models.ImageSlice) RecentlyAddedImages { } var deviceFound bool for i, ra := range r { - if ra.Device.ID == image.R.Device.ID { + if ra.Device.Slug == image.R.Device.Slug { deviceFound = true var subredditFound bool for j, subreddit := range r[i].Subreddits { - if subreddit.Subreddit.ID == image.R.Subreddit.ID { + if subreddit.Subreddit.Name == image.R.Subreddit.Name { subredditFound = true r[i].Subreddits[j].Images = append(r[i].Subreddits[j].Images, image) count++ @@ -89,7 +88,5 @@ func NewRecentlyAddedImages(images models.ImageSlice) RecentlyAddedImages { return strings.Compare(leftName, rightName) }) - fmt.Println("image count", count) - return r } diff --git a/views/homeview/recently_added_image.templ b/views/homeview/recently_added_image.templ index 23a3539..87e99bc 100644 --- a/views/homeview/recently_added_image.templ +++ b/views/homeview/recently_added_image.templ @@ -28,7 +28,7 @@ templ RecentlyAddedImageCard(data *models.Image, opts ImageCardOption) { { @@ -37,9 +37,9 @@ templ RecentlyAddedImageCard(data *models.Image, opts ImageCardOption) { { truncateTitle(data.Title) } + >{ truncateTitle(data.PostTitle) } } - { data.Poster } + { data.PostAuthor }
@utils.RelativeTimeNode(fmt.Sprintf("relative-time-%s", data.PostName), data.CreatedAt)