api: implemented download images
This commit is contained in:
parent
f031223150
commit
640fe31d10
24
api/api.go
24
api/api.go
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/robfig/cron/v3"
|
||||
"github.com/teivah/broadcast"
|
||||
"github.com/tigorlazuardi/redmage/api/bmessage"
|
||||
"github.com/tigorlazuardi/redmage/api/reddit"
|
||||
"github.com/tigorlazuardi/redmage/config"
|
||||
"github.com/tigorlazuardi/redmage/db/queries"
|
||||
"github.com/tigorlazuardi/redmage/pkg/errs"
|
||||
|
@ -24,16 +25,31 @@ type API struct {
|
|||
downloadBroadcast *broadcast.Relay[bmessage.ImageDownloadMessage]
|
||||
|
||||
config *config.Config
|
||||
|
||||
imageSemaphore chan struct{}
|
||||
subredditSemaphore chan struct{}
|
||||
|
||||
reddit *reddit.Reddit
|
||||
}
|
||||
|
||||
func New(q *queries.Queries, db *sql.DB, cfg *config.Config) *API {
|
||||
type Dependencies struct {
|
||||
Queries *queries.Queries
|
||||
DB *sql.DB
|
||||
Config *config.Config
|
||||
Reddit *reddit.Reddit
|
||||
}
|
||||
|
||||
func New(deps Dependencies) *API {
|
||||
return &API{
|
||||
queries: q,
|
||||
db: db,
|
||||
queries: deps.Queries,
|
||||
db: deps.DB,
|
||||
scheduler: cron.New(),
|
||||
scheduleMap: make(map[cron.EntryID]queries.Subreddit, 8),
|
||||
downloadBroadcast: broadcast.NewRelay[bmessage.ImageDownloadMessage](),
|
||||
config: cfg,
|
||||
config: deps.Config,
|
||||
imageSemaphore: make(chan struct{}, deps.Config.Int("download.concurrency.images")),
|
||||
subredditSemaphore: make(chan struct{}, deps.Config.Int("download.concurrency.subreddits")),
|
||||
reddit: deps.Reddit,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@ import (
|
|||
type ImageMetadata struct {
|
||||
Kind ImageKind
|
||||
URL string
|
||||
Height int
|
||||
Width int
|
||||
Height int64
|
||||
Width int64
|
||||
}
|
||||
|
||||
type ImageKind int
|
||||
|
@ -18,7 +18,36 @@ const (
|
|||
KindThumbnail
|
||||
)
|
||||
|
||||
type DownloadEvent int
|
||||
|
||||
func (do DownloadEvent) MarshalJSON() ([]byte, error) {
|
||||
return []byte(`"` + do.String() + `"`), nil
|
||||
}
|
||||
|
||||
func (do DownloadEvent) String() string {
|
||||
switch do {
|
||||
case DownloadStart:
|
||||
return "DownloadStart"
|
||||
case DownloadProgress:
|
||||
return "DownloadProgress"
|
||||
case DownloadEnd:
|
||||
return "DownloadEnd"
|
||||
case DownloadError:
|
||||
return "DownloadError"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DownloadStart DownloadEvent = iota
|
||||
DownloadProgress
|
||||
DownloadEnd
|
||||
DownloadError
|
||||
)
|
||||
|
||||
type ImageDownloadMessage struct {
|
||||
Event DownloadEvent
|
||||
Metadata ImageMetadata
|
||||
ContentLength units.MetricBytes
|
||||
Downloaded units.MetricBytes
|
||||
|
|
|
@ -3,17 +3,30 @@ package api
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/tigorlazuardi/redmage/api/reddit"
|
||||
"github.com/tigorlazuardi/redmage/db/queries"
|
||||
"github.com/tigorlazuardi/redmage/pkg/errs"
|
||||
"github.com/tigorlazuardi/redmage/pkg/log"
|
||||
"github.com/tigorlazuardi/redmage/pkg/telemetry"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type DownloadSubredditParams struct {
|
||||
Countback int
|
||||
NSFW bool
|
||||
Devices []queries.Device
|
||||
Type int
|
||||
SubredditType reddit.SubredditType
|
||||
}
|
||||
|
||||
var (
|
||||
|
@ -31,5 +44,249 @@ func (api *API) DownloadSubredditImages(ctx context.Context, subredditName strin
|
|||
return errs.Wrapw(ErrNoDevices, "downloading images requires at least one device configured").Code(http.StatusBadRequest)
|
||||
}
|
||||
|
||||
ctx, span := tracer.Start(ctx, "*API.DownloadSubredditImages", trace.WithAttributes(attribute.String("subreddit", subredditName)))
|
||||
defer span.End()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
countback := params.Countback
|
||||
|
||||
for page := 1; countback > 0; page += 1 {
|
||||
limit := countback
|
||||
if limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
list, err := api.reddit.GetPosts(ctx, reddit.GetPostsParam{
|
||||
Subreddit: subredditName,
|
||||
Limit: limit,
|
||||
Page: page,
|
||||
SubredditType: params.SubredditType,
|
||||
})
|
||||
if err != nil {
|
||||
return errs.Wrapw(err, "failed to get posts", "subreddit_name", subredditName, "params", params)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(ctx context.Context, posts reddit.Listing) {
|
||||
defer wg.Done()
|
||||
err := api.downloadSubredditListImage(ctx, list, params)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to download image")
|
||||
}
|
||||
}(ctx, list)
|
||||
countback -= len(list.GetPosts())
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.Listing, params DownloadSubredditParams) error {
|
||||
ctx, span := tracer.Start(ctx, "*API.downloadSubredditImage")
|
||||
defer span.End()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
for _, post := range list.GetPosts() {
|
||||
if !post.IsImagePost() {
|
||||
continue
|
||||
}
|
||||
devices := getDevicesThatAcceptPost(post, params.Devices)
|
||||
if len(devices) == 0 {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
api.imageSemaphore <- struct{}{}
|
||||
go func(ctx context.Context, post reddit.Post) {
|
||||
defer func() {
|
||||
<-api.imageSemaphore
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
imageHandler, err := api.reddit.DownloadImage(ctx, post, api.downloadBroadcast)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to download image")
|
||||
return
|
||||
}
|
||||
defer imageHandler.Close()
|
||||
|
||||
// copy to temp dir first to avoid copying incomplete files.
|
||||
tmpImageFile, err := api.copyImageToTempDir(ctx, imageHandler)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to download image to temp file")
|
||||
return
|
||||
}
|
||||
defer tmpImageFile.Close()
|
||||
|
||||
w, close, err := api.createDeviceImageWriters(post, devices)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to create image files")
|
||||
return
|
||||
}
|
||||
defer close()
|
||||
_, err = io.Copy(w, tmpImageFile)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to create save image files")
|
||||
return
|
||||
}
|
||||
thumbnailPath := post.GetThumbnailTargetPath(api.config)
|
||||
_, errStat := os.Stat(thumbnailPath)
|
||||
if errStat == nil {
|
||||
// file exist
|
||||
return
|
||||
}
|
||||
if !errors.Is(errStat, os.ErrNotExist) {
|
||||
log.New(ctx).Err(err).Error("failed to check thumbail existence", "path", thumbnailPath)
|
||||
return
|
||||
}
|
||||
|
||||
thumbnailSource, err := imaging.Open(tmpImageFile.filename)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to open temp thumbnail file", "filename", tmpImageFile.filename)
|
||||
return
|
||||
}
|
||||
|
||||
thumbnail := imaging.Resize(thumbnailSource, 256, 0, imaging.Lanczos)
|
||||
thumbnailFile, err := os.Create(thumbnailPath)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to create thumbnail file", "filename", thumbnailPath)
|
||||
return
|
||||
}
|
||||
defer thumbnailFile.Close()
|
||||
|
||||
err = jpeg.Encode(thumbnailFile, thumbnail, nil)
|
||||
if err != nil {
|
||||
log.New(ctx).Err(err).Error("failed to encode thumbnail file to jpeg", "filename", thumbnailPath)
|
||||
return
|
||||
}
|
||||
}(ctx, post)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (api *API) createDeviceImageWriters(post reddit.Post, devices []queries.Device) (writer io.Writer, close func(), err error) {
|
||||
// open file for each device
|
||||
var files []*os.File
|
||||
var writers []io.Writer
|
||||
for _, device := range devices {
|
||||
var filename string
|
||||
if device.WindowsWallpaperMode == 1 {
|
||||
filename = post.GetWindowsWallpaperImageTargetPath(api.config, device)
|
||||
} else {
|
||||
filename = post.GetImageTargetPath(api.config, device)
|
||||
}
|
||||
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
for _, f := range files {
|
||||
_ = f.Close()
|
||||
}
|
||||
return nil, nil, errs.Wrapw(err, "failed to open temp image file",
|
||||
"device_name", device.Name,
|
||||
"filename", filename,
|
||||
)
|
||||
}
|
||||
files = append(files, file)
|
||||
writers = append(writers, file)
|
||||
}
|
||||
|
||||
return io.MultiWriter(writers...), func() {
|
||||
for _, file := range files {
|
||||
_ = file.Close()
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getDevicesThatAcceptPost(post reddit.Post, devices []queries.Device) []queries.Device {
|
||||
var devs []queries.Device
|
||||
for _, device := range devices {
|
||||
if shouldDownloadPostForDevice(post, device) {
|
||||
devs = append(devices, device)
|
||||
}
|
||||
}
|
||||
return devs
|
||||
}
|
||||
|
||||
func shouldDownloadPostForDevice(post reddit.Post, device queries.Device) bool {
|
||||
if post.IsNSFW() && device.Nsfw == 0 {
|
||||
return false
|
||||
}
|
||||
if math.Abs(deviceAspectRatio(device)-post.GetImageAspectRatio()) > device.AspectRatioTolerance { // outside of aspect ratio tolerance
|
||||
return false
|
||||
}
|
||||
width, height := post.GetImageSize()
|
||||
if device.MaxX > 0 && width > device.MaxX {
|
||||
return false
|
||||
}
|
||||
if device.MaxY > 0 && height > device.MaxY {
|
||||
return false
|
||||
}
|
||||
if device.MinX > 0 && width < device.MinX {
|
||||
return false
|
||||
}
|
||||
if device.MinY > 0 && height < device.MinY {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func deviceAspectRatio(device queries.Device) float64 {
|
||||
return float64(device.ResolutionX) / float64(device.ResolutionY)
|
||||
}
|
||||
|
||||
type tempFile struct {
|
||||
filename string
|
||||
file *os.File
|
||||
}
|
||||
|
||||
func (te *tempFile) Read(p []byte) (n int, err error) {
|
||||
return te.file.Read(p)
|
||||
}
|
||||
|
||||
func (te *tempFile) Close() error {
|
||||
return te.file.Close()
|
||||
}
|
||||
|
||||
// copyImageToTempDir copies the image to a temporary directory and returns the file handle
|
||||
//
|
||||
// file must be closed by the caller after use.
|
||||
//
|
||||
// file is nil if an error occurred.
|
||||
func (api *API) copyImageToTempDir(ctx context.Context, img reddit.PostImage) (tmp *tempFile, err error) {
|
||||
_, span := tracer.Start(ctx, "*API.copyImageToTempDir")
|
||||
defer func() { telemetry.EndWithStatus(span, err) }()
|
||||
|
||||
// ignore error because url is always valid if this
|
||||
// function is called
|
||||
url, _ := url.Parse(img.URL)
|
||||
|
||||
split := strings.Split(url.Path, "/")
|
||||
imageFilename := split[len(split)-1]
|
||||
tmpDirname := path.Join(os.TempDir(), "redmage")
|
||||
_ = os.MkdirAll(tmpDirname, 0644)
|
||||
tmpFilename := path.Join(tmpDirname, imageFilename)
|
||||
|
||||
file, err := os.OpenFile(tmpFilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return nil, errs.Wrapw(err, "failed to open temp image file",
|
||||
"temp_file_path", tmpFilename,
|
||||
"image_url", img.URL,
|
||||
)
|
||||
}
|
||||
|
||||
_, err = io.Copy(file, img.File)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, errs.Wrapw(err, "failed to download image to temp file",
|
||||
"temp_file_path", tmpFilename,
|
||||
"image_url", img.URL,
|
||||
)
|
||||
}
|
||||
|
||||
return &tempFile{
|
||||
file: file,
|
||||
filename: tmpFilename,
|
||||
}, err
|
||||
}
|
||||
|
|
|
@ -8,7 +8,6 @@ import (
|
|||
"github.com/alecthomas/units"
|
||||
"github.com/tigorlazuardi/redmage/api/bmessage"
|
||||
"github.com/tigorlazuardi/redmage/pkg/errs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type DownloadStatusBroadcaster interface {
|
||||
|
@ -20,38 +19,42 @@ type NullDownloadStatusBroadcaster struct{}
|
|||
func (NullDownloadStatusBroadcaster) Broadcast(bmessage.ImageDownloadMessage) {}
|
||||
|
||||
type PostImage struct {
|
||||
ImageURL string
|
||||
ImageFile io.Reader
|
||||
ThumbnailURL string
|
||||
ThumbnailFile io.Reader
|
||||
URL string
|
||||
File io.ReadCloser
|
||||
}
|
||||
|
||||
func (po *PostImage) Read(p []byte) (n int, err error) {
|
||||
return po.File.Read(p)
|
||||
}
|
||||
|
||||
func (po *PostImage) Close() error {
|
||||
return po.File.Close()
|
||||
}
|
||||
|
||||
// DownloadImage downloades the image.
|
||||
//
|
||||
// If downloading image or thumbnail fails
|
||||
func (reddit *Reddit) DownloadImage(ctx context.Context, post Post, broadcaster DownloadStatusBroadcaster) (image PostImage, err error) {
|
||||
imageUrl, thumbnailUrl := post.GetImageURL(), post.GetThumbnailURL()
|
||||
image.ImageURL = imageUrl
|
||||
image.ThumbnailURL = thumbnailUrl
|
||||
imageUrl := post.GetImageURL()
|
||||
image.URL = imageUrl
|
||||
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
image.ImageFile, err = reddit.downloadImage(groupCtx, post, bmessage.KindImage, broadcaster)
|
||||
return err
|
||||
})
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
image.ThumbnailFile, err = reddit.downloadImage(groupCtx, post, bmessage.KindThumbnail, broadcaster)
|
||||
return err
|
||||
})
|
||||
|
||||
err = group.Wait()
|
||||
image.File, err = reddit.downloadImage(ctx, post, bmessage.KindImage, broadcaster)
|
||||
return image, err
|
||||
}
|
||||
|
||||
func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessage.ImageKind, broadcaster DownloadStatusBroadcaster) (io.Reader, error) {
|
||||
func (reddit *Reddit) DownloadThumbnail(ctx context.Context, post Post, broadcaster DownloadStatusBroadcaster) (image PostImage, err error) {
|
||||
imageUrl := post.GetThumbnailURL()
|
||||
image.URL = imageUrl
|
||||
|
||||
image.File, err = reddit.downloadImage(ctx, post, bmessage.KindThumbnail, broadcaster)
|
||||
return image, err
|
||||
}
|
||||
|
||||
func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessage.ImageKind, broadcaster DownloadStatusBroadcaster) (io.ReadCloser, error) {
|
||||
var (
|
||||
url string
|
||||
height int
|
||||
width int
|
||||
height int64
|
||||
width int64
|
||||
)
|
||||
if kind == bmessage.KindImage {
|
||||
url = post.GetImageURL()
|
||||
|
@ -75,15 +78,23 @@ func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessag
|
|||
if metricSpeed == 0 {
|
||||
metricSpeed = 10 * units.KB
|
||||
}
|
||||
idr := &ImageDownloadReader{
|
||||
OnProgress: func(downloaded int64, contentLength int64, err error) {
|
||||
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
|
||||
Metadata: bmessage.ImageMetadata{
|
||||
metadata := bmessage.ImageMetadata{
|
||||
URL: url,
|
||||
Height: height,
|
||||
Width: width,
|
||||
Kind: kind,
|
||||
},
|
||||
}
|
||||
idr := &ImageDownloadReader{
|
||||
OnProgress: func(downloaded int64, contentLength int64, err error) {
|
||||
var event bmessage.DownloadEvent
|
||||
if err != nil {
|
||||
event = bmessage.DownloadError
|
||||
} else {
|
||||
event = bmessage.DownloadProgress
|
||||
}
|
||||
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
|
||||
Event: event,
|
||||
Metadata: metadata,
|
||||
ContentLength: units.MetricBytes(resp.ContentLength),
|
||||
Downloaded: units.MetricBytes(downloaded),
|
||||
Subreddit: post.GetSubreddit(),
|
||||
|
@ -92,6 +103,18 @@ func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessag
|
|||
Error: err,
|
||||
})
|
||||
},
|
||||
OnClose: func(downloaded, contentLength int64, closeErr error) {
|
||||
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
|
||||
Event: bmessage.DownloadEnd,
|
||||
Metadata: metadata,
|
||||
ContentLength: units.MetricBytes(resp.ContentLength),
|
||||
Downloaded: units.MetricBytes(downloaded),
|
||||
Subreddit: post.GetSubreddit(),
|
||||
PostURL: post.GetPermalink(),
|
||||
PostID: post.GetID(),
|
||||
Error: closeErr,
|
||||
})
|
||||
},
|
||||
IdleTimeout: reddit.Config.Duration("download.timeout.idle"),
|
||||
IdleSpeedThreshold: metricSpeed,
|
||||
}
|
||||
|
@ -100,6 +123,18 @@ func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessag
|
|||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
defer resp.Body.Close()
|
||||
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
|
||||
Event: bmessage.DownloadStart,
|
||||
Metadata: bmessage.ImageMetadata{
|
||||
URL: url,
|
||||
Height: height,
|
||||
Width: width,
|
||||
Kind: kind,
|
||||
},
|
||||
Subreddit: post.GetSubreddit(),
|
||||
PostURL: post.GetPermalink(),
|
||||
PostID: post.GetID(),
|
||||
})
|
||||
_, err := io.Copy(writer, resp.Body)
|
||||
_ = writer.CloseWithError(err)
|
||||
}()
|
||||
|
|
|
@ -53,7 +53,7 @@ func (reddit *Reddit) GetPosts(ctx context.Context, params GetPostsParam) (posts
|
|||
return posts, errs.Fail("reddit: unexpected status code when executing GetPosts",
|
||||
slog.Group("request", "url", url, "params", params),
|
||||
slog.Group("response", "status_code", res.StatusCode, "body", formatLogBody(res, body)),
|
||||
)
|
||||
).Code(res.StatusCode)
|
||||
}
|
||||
|
||||
err = json.NewDecoder(res.Body).Decode(&posts)
|
||||
|
|
|
@ -15,7 +15,7 @@ var ErrIdleTimeoutReached = errors.New("download idle timeout reached")
|
|||
|
||||
type ImageDownloadReader struct {
|
||||
OnProgress func(downloaded int64, contentLength int64, err error)
|
||||
OnClose func(closeErr error)
|
||||
OnClose func(downloaded int64, contentLength int64, closeErr error)
|
||||
IdleTimeout time.Duration
|
||||
IdleSpeedThreshold units.MetricBytes
|
||||
|
||||
|
@ -107,7 +107,7 @@ func (idr *ImageDownloadReader) Close() error {
|
|||
idr.exit <- struct{}{}
|
||||
err := idr.reader.Close()
|
||||
if idr.OnClose != nil {
|
||||
idr.OnClose(err)
|
||||
idr.OnClose(idr.downloaded.Load(), idr.contentLength, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -1,5 +1,15 @@
|
|||
package reddit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/tigorlazuardi/redmage/config"
|
||||
"github.com/tigorlazuardi/redmage/db/queries"
|
||||
)
|
||||
|
||||
type Listing struct {
|
||||
Kind string `json:"kind"`
|
||||
Data Data `json:"data"`
|
||||
|
@ -15,8 +25,8 @@ type (
|
|||
Gildings struct{}
|
||||
Source struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Width int64 `json:"width"`
|
||||
Height int64 `json:"height"`
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -86,7 +96,7 @@ type PostData struct {
|
|||
Pwls int `json:"pwls"`
|
||||
LinkFlairCSSClass string `json:"link_flair_css_class"`
|
||||
Downs int `json:"downs"`
|
||||
ThumbnailHeight int `json:"thumbnail_height"`
|
||||
ThumbnailHeight int64 `json:"thumbnail_height"`
|
||||
TopAwardedType any `json:"top_awarded_type"`
|
||||
HideScore bool `json:"hide_score"`
|
||||
MediaMetadata map[string]MediaMetadata `json:"media_metadata"`
|
||||
|
@ -98,7 +108,7 @@ type PostData struct {
|
|||
Ups int `json:"ups"`
|
||||
Domain string `json:"domain"`
|
||||
MediaEmbed MediaEmbed `json:"media_embed"`
|
||||
ThumbnailWidth int `json:"thumbnail_width"`
|
||||
ThumbnailWidth int64 `json:"thumbnail_width"`
|
||||
AuthorFlairTemplateID string `json:"author_flair_template_id"`
|
||||
IsOriginalContent bool `json:"is_original_content"`
|
||||
UserReports []any `json:"user_reports"`
|
||||
|
@ -189,6 +199,10 @@ type Post struct {
|
|||
Data PostData `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func (post *Post) IsNSFW() bool {
|
||||
return post.Data.Over18
|
||||
}
|
||||
|
||||
func (post *Post) IsImagePost() bool {
|
||||
return post.Data.PostHint == "image"
|
||||
}
|
||||
|
@ -197,7 +211,44 @@ func (post *Post) GetImageURL() string {
|
|||
return post.Data.URL
|
||||
}
|
||||
|
||||
func (post *Post) GetImageSize() (width, height int) {
|
||||
func (post *Post) GetImageAspectRatio() float64 {
|
||||
width, height := post.GetImageSize()
|
||||
if height == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(width) / float64(height)
|
||||
}
|
||||
|
||||
func (post *Post) GetImageTargetPath(cfg *config.Config, device queries.Device) string {
|
||||
baseDownloadDir := cfg.String("download.directory")
|
||||
return path.Join(baseDownloadDir, device.Name, post.GetSubreddit(), post.GetImageFilename())
|
||||
}
|
||||
|
||||
func (post *Post) GetWindowsWallpaperImageTargetPath(cfg *config.Config, device queries.Device) string {
|
||||
baseDownloadDir := cfg.String("download.directory")
|
||||
filename := fmt.Sprintf("%s_%s", post.GetSubreddit(), post.GetImageFilename())
|
||||
return path.Join(baseDownloadDir, device.Name, filename)
|
||||
}
|
||||
|
||||
func (post *Post) GetThumbnailTargetPath(cfg *config.Config) string {
|
||||
baseDownloadDir := cfg.String("download.directory")
|
||||
return path.Join(baseDownloadDir, "_thumbnails", post.GetSubreddit(), post.GetImageFilename())
|
||||
}
|
||||
|
||||
func (post *Post) GetImageFilename() string {
|
||||
if !post.IsImagePost() {
|
||||
return ""
|
||||
}
|
||||
link := post.GetImageURL()
|
||||
u, _ := url.Parse(link)
|
||||
if u == nil {
|
||||
return ""
|
||||
}
|
||||
split := strings.Split(u.Path, "/")
|
||||
return split[len(split)-1]
|
||||
}
|
||||
|
||||
func (post *Post) GetImageSize() (width, height int64) {
|
||||
if len(post.Data.Preview.Images) == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
|
@ -209,7 +260,7 @@ func (post *Post) GetThumbnailURL() string {
|
|||
return post.Data.Thumbnail
|
||||
}
|
||||
|
||||
func (post *Post) GetThumbnailSize() (width, height int) {
|
||||
func (post *Post) GetThumbnailSize() (width, height int64) {
|
||||
return post.Data.ThumbnailWidth, post.Data.ThumbnailHeight
|
||||
}
|
||||
|
||||
|
|
14
cli/serve.go
14
cli/serve.go
|
@ -2,10 +2,12 @@ package cli
|
|||
|
||||
import (
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/tigorlazuardi/redmage/api"
|
||||
"github.com/tigorlazuardi/redmage/api/reddit"
|
||||
"github.com/tigorlazuardi/redmage/db"
|
||||
"github.com/tigorlazuardi/redmage/db/queries"
|
||||
"github.com/tigorlazuardi/redmage/pkg/log"
|
||||
|
@ -35,7 +37,17 @@ var serveCmd = &cobra.Command{
|
|||
|
||||
queries := queries.New(db)
|
||||
|
||||
api := api.New(queries, db, cfg)
|
||||
red := &reddit.Reddit{
|
||||
Client: http.DefaultClient,
|
||||
Config: cfg,
|
||||
}
|
||||
|
||||
api := api.New(api.Dependencies{
|
||||
Queries: queries,
|
||||
DB: db,
|
||||
Config: cfg,
|
||||
Reddit: red,
|
||||
})
|
||||
|
||||
server := server.New(cfg, api, PublicDir)
|
||||
|
||||
|
|
|
@ -15,7 +15,8 @@ var DefaultConfig = map[string]any{
|
|||
"db.string": "data.db",
|
||||
"db.automigrate": true,
|
||||
|
||||
"download.concurrency": 5,
|
||||
"download.concurrency.images": 5,
|
||||
"download.concurrency.subreddits": 3,
|
||||
"download.directory": "",
|
||||
"download.timeout.headers": "10s",
|
||||
"download.timeout.idle": "5s",
|
||||
|
|
|
@ -11,6 +11,7 @@ CREATE TABLE devices(
|
|||
max_x INTEGER NOT NULL DEFAULT 0,
|
||||
max_y INTEGER NOT NULL DEFAULT 0,
|
||||
nsfw INTEGER NOT NULL DEFAULT 0,
|
||||
windows_wallpaper_mode INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
|
||||
);
|
||||
|
|
2
go.mod
2
go.mod
|
@ -34,6 +34,7 @@ require (
|
|||
github.com/samber/lo v1.38.1 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect
|
||||
golang.org/x/net v0.23.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
|
||||
|
@ -44,6 +45,7 @@ require (
|
|||
|
||||
require (
|
||||
github.com/XSAM/otelsql v0.29.0
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/go-logr/logr v1.4.1 // indirect
|
||||
|
|
5
go.sum
5
go.sum
|
@ -33,6 +33,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
||||
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
||||
github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
|
||||
github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
|
||||
|
@ -248,6 +250,8 @@ golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
|||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||
golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE=
|
||||
golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
|
||||
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
|
||||
|
@ -264,6 +268,7 @@ golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
|||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
|
||||
|
|
31
pkg/telemetry/status.go
Normal file
31
pkg/telemetry/status.go
Normal file
|
@ -0,0 +1,31 @@
|
|||
package telemetry
|
||||
|
||||
import (
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// EndWithStatus ends the span with the status of the error if not nil
|
||||
// otherwise it will set the status to OK.
|
||||
//
|
||||
// This function should be used for ending spans, not for starting spans
|
||||
// or spans that will have children, to avoid duplicate error recordings.
|
||||
//
|
||||
// Do not defer this function directly since err might be nil at the
|
||||
// start of defer call. Instead it should be wrapped in a function to
|
||||
// capture the error correctly.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// var err error
|
||||
// ctx, span := tracer.Start(ctx, "my-operation")
|
||||
// defer func() { telemetry.EndWithStatus(span, err) }()
|
||||
func EndWithStatus(span trace.Span, err error) {
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
} else {
|
||||
span.SetStatus(codes.Ok, "")
|
||||
}
|
||||
span.End()
|
||||
}
|
Loading…
Reference in a new issue