api: implemented download images

This commit is contained in:
Tigor Hutasuhut 2024-04-14 00:32:55 +07:00
parent f031223150
commit 640fe31d10
13 changed files with 498 additions and 58 deletions

View file

@ -8,6 +8,7 @@ import (
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"github.com/teivah/broadcast" "github.com/teivah/broadcast"
"github.com/tigorlazuardi/redmage/api/bmessage" "github.com/tigorlazuardi/redmage/api/bmessage"
"github.com/tigorlazuardi/redmage/api/reddit"
"github.com/tigorlazuardi/redmage/config" "github.com/tigorlazuardi/redmage/config"
"github.com/tigorlazuardi/redmage/db/queries" "github.com/tigorlazuardi/redmage/db/queries"
"github.com/tigorlazuardi/redmage/pkg/errs" "github.com/tigorlazuardi/redmage/pkg/errs"
@ -24,16 +25,31 @@ type API struct {
downloadBroadcast *broadcast.Relay[bmessage.ImageDownloadMessage] downloadBroadcast *broadcast.Relay[bmessage.ImageDownloadMessage]
config *config.Config config *config.Config
imageSemaphore chan struct{}
subredditSemaphore chan struct{}
reddit *reddit.Reddit
} }
func New(q *queries.Queries, db *sql.DB, cfg *config.Config) *API { type Dependencies struct {
Queries *queries.Queries
DB *sql.DB
Config *config.Config
Reddit *reddit.Reddit
}
func New(deps Dependencies) *API {
return &API{ return &API{
queries: q, queries: deps.Queries,
db: db, db: deps.DB,
scheduler: cron.New(), scheduler: cron.New(),
scheduleMap: make(map[cron.EntryID]queries.Subreddit, 8), scheduleMap: make(map[cron.EntryID]queries.Subreddit, 8),
downloadBroadcast: broadcast.NewRelay[bmessage.ImageDownloadMessage](), downloadBroadcast: broadcast.NewRelay[bmessage.ImageDownloadMessage](),
config: cfg, config: deps.Config,
imageSemaphore: make(chan struct{}, deps.Config.Int("download.concurrency.images")),
subredditSemaphore: make(chan struct{}, deps.Config.Int("download.concurrency.subreddits")),
reddit: deps.Reddit,
} }
} }

View file

@ -7,8 +7,8 @@ import (
type ImageMetadata struct { type ImageMetadata struct {
Kind ImageKind Kind ImageKind
URL string URL string
Height int Height int64
Width int Width int64
} }
type ImageKind int type ImageKind int
@ -18,7 +18,36 @@ const (
KindThumbnail KindThumbnail
) )
type DownloadEvent int
func (do DownloadEvent) MarshalJSON() ([]byte, error) {
return []byte(`"` + do.String() + `"`), nil
}
func (do DownloadEvent) String() string {
switch do {
case DownloadStart:
return "DownloadStart"
case DownloadProgress:
return "DownloadProgress"
case DownloadEnd:
return "DownloadEnd"
case DownloadError:
return "DownloadError"
default:
return "Unknown"
}
}
const (
DownloadStart DownloadEvent = iota
DownloadProgress
DownloadEnd
DownloadError
)
type ImageDownloadMessage struct { type ImageDownloadMessage struct {
Event DownloadEvent
Metadata ImageMetadata Metadata ImageMetadata
ContentLength units.MetricBytes ContentLength units.MetricBytes
Downloaded units.MetricBytes Downloaded units.MetricBytes

View file

@ -3,17 +3,30 @@ package api
import ( import (
"context" "context"
"errors" "errors"
"image/jpeg"
"io"
"math"
"net/http" "net/http"
"net/url"
"os"
"path"
"strings"
"sync"
"github.com/disintegration/imaging"
"github.com/tigorlazuardi/redmage/api/reddit"
"github.com/tigorlazuardi/redmage/db/queries" "github.com/tigorlazuardi/redmage/db/queries"
"github.com/tigorlazuardi/redmage/pkg/errs" "github.com/tigorlazuardi/redmage/pkg/errs"
"github.com/tigorlazuardi/redmage/pkg/log"
"github.com/tigorlazuardi/redmage/pkg/telemetry"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
) )
type DownloadSubredditParams struct { type DownloadSubredditParams struct {
Countback int Countback int
NSFW bool
Devices []queries.Device Devices []queries.Device
Type int SubredditType reddit.SubredditType
} }
var ( var (
@ -31,5 +44,249 @@ func (api *API) DownloadSubredditImages(ctx context.Context, subredditName strin
return errs.Wrapw(ErrNoDevices, "downloading images requires at least one device configured").Code(http.StatusBadRequest) return errs.Wrapw(ErrNoDevices, "downloading images requires at least one device configured").Code(http.StatusBadRequest)
} }
ctx, span := tracer.Start(ctx, "*API.DownloadSubredditImages", trace.WithAttributes(attribute.String("subreddit", subredditName)))
defer span.End()
wg := sync.WaitGroup{}
countback := params.Countback
for page := 1; countback > 0; page += 1 {
limit := countback
if limit > 100 {
limit = 100
}
list, err := api.reddit.GetPosts(ctx, reddit.GetPostsParam{
Subreddit: subredditName,
Limit: limit,
Page: page,
SubredditType: params.SubredditType,
})
if err != nil {
return errs.Wrapw(err, "failed to get posts", "subreddit_name", subredditName, "params", params)
}
wg.Add(1)
go func(ctx context.Context, posts reddit.Listing) {
defer wg.Done()
err := api.downloadSubredditListImage(ctx, list, params)
if err != nil {
log.New(ctx).Err(err).Error("failed to download image")
}
}(ctx, list)
countback -= len(list.GetPosts())
}
wg.Wait()
return nil return nil
} }
func (api *API) downloadSubredditListImage(ctx context.Context, list reddit.Listing, params DownloadSubredditParams) error {
ctx, span := tracer.Start(ctx, "*API.downloadSubredditImage")
defer span.End()
wg := sync.WaitGroup{}
for _, post := range list.GetPosts() {
if !post.IsImagePost() {
continue
}
devices := getDevicesThatAcceptPost(post, params.Devices)
if len(devices) == 0 {
continue
}
wg.Add(1)
api.imageSemaphore <- struct{}{}
go func(ctx context.Context, post reddit.Post) {
defer func() {
<-api.imageSemaphore
wg.Done()
}()
imageHandler, err := api.reddit.DownloadImage(ctx, post, api.downloadBroadcast)
if err != nil {
log.New(ctx).Err(err).Error("failed to download image")
return
}
defer imageHandler.Close()
// copy to temp dir first to avoid copying incomplete files.
tmpImageFile, err := api.copyImageToTempDir(ctx, imageHandler)
if err != nil {
log.New(ctx).Err(err).Error("failed to download image to temp file")
return
}
defer tmpImageFile.Close()
w, close, err := api.createDeviceImageWriters(post, devices)
if err != nil {
log.New(ctx).Err(err).Error("failed to create image files")
return
}
defer close()
_, err = io.Copy(w, tmpImageFile)
if err != nil {
log.New(ctx).Err(err).Error("failed to create save image files")
return
}
thumbnailPath := post.GetThumbnailTargetPath(api.config)
_, errStat := os.Stat(thumbnailPath)
if errStat == nil {
// file exist
return
}
if !errors.Is(errStat, os.ErrNotExist) {
log.New(ctx).Err(err).Error("failed to check thumbail existence", "path", thumbnailPath)
return
}
thumbnailSource, err := imaging.Open(tmpImageFile.filename)
if err != nil {
log.New(ctx).Err(err).Error("failed to open temp thumbnail file", "filename", tmpImageFile.filename)
return
}
thumbnail := imaging.Resize(thumbnailSource, 256, 0, imaging.Lanczos)
thumbnailFile, err := os.Create(thumbnailPath)
if err != nil {
log.New(ctx).Err(err).Error("failed to create thumbnail file", "filename", thumbnailPath)
return
}
defer thumbnailFile.Close()
err = jpeg.Encode(thumbnailFile, thumbnail, nil)
if err != nil {
log.New(ctx).Err(err).Error("failed to encode thumbnail file to jpeg", "filename", thumbnailPath)
return
}
}(ctx, post)
}
wg.Wait()
return nil
}
func (api *API) createDeviceImageWriters(post reddit.Post, devices []queries.Device) (writer io.Writer, close func(), err error) {
// open file for each device
var files []*os.File
var writers []io.Writer
for _, device := range devices {
var filename string
if device.WindowsWallpaperMode == 1 {
filename = post.GetWindowsWallpaperImageTargetPath(api.config, device)
} else {
filename = post.GetImageTargetPath(api.config, device)
}
file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
for _, f := range files {
_ = f.Close()
}
return nil, nil, errs.Wrapw(err, "failed to open temp image file",
"device_name", device.Name,
"filename", filename,
)
}
files = append(files, file)
writers = append(writers, file)
}
return io.MultiWriter(writers...), func() {
for _, file := range files {
_ = file.Close()
}
}, nil
}
func getDevicesThatAcceptPost(post reddit.Post, devices []queries.Device) []queries.Device {
var devs []queries.Device
for _, device := range devices {
if shouldDownloadPostForDevice(post, device) {
devs = append(devices, device)
}
}
return devs
}
func shouldDownloadPostForDevice(post reddit.Post, device queries.Device) bool {
if post.IsNSFW() && device.Nsfw == 0 {
return false
}
if math.Abs(deviceAspectRatio(device)-post.GetImageAspectRatio()) > device.AspectRatioTolerance { // outside of aspect ratio tolerance
return false
}
width, height := post.GetImageSize()
if device.MaxX > 0 && width > device.MaxX {
return false
}
if device.MaxY > 0 && height > device.MaxY {
return false
}
if device.MinX > 0 && width < device.MinX {
return false
}
if device.MinY > 0 && height < device.MinY {
return false
}
return true
}
func deviceAspectRatio(device queries.Device) float64 {
return float64(device.ResolutionX) / float64(device.ResolutionY)
}
type tempFile struct {
filename string
file *os.File
}
func (te *tempFile) Read(p []byte) (n int, err error) {
return te.file.Read(p)
}
func (te *tempFile) Close() error {
return te.file.Close()
}
// copyImageToTempDir copies the image to a temporary directory and returns the file handle
//
// file must be closed by the caller after use.
//
// file is nil if an error occurred.
func (api *API) copyImageToTempDir(ctx context.Context, img reddit.PostImage) (tmp *tempFile, err error) {
_, span := tracer.Start(ctx, "*API.copyImageToTempDir")
defer func() { telemetry.EndWithStatus(span, err) }()
// ignore error because url is always valid if this
// function is called
url, _ := url.Parse(img.URL)
split := strings.Split(url.Path, "/")
imageFilename := split[len(split)-1]
tmpDirname := path.Join(os.TempDir(), "redmage")
_ = os.MkdirAll(tmpDirname, 0644)
tmpFilename := path.Join(tmpDirname, imageFilename)
file, err := os.OpenFile(tmpFilename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return nil, errs.Wrapw(err, "failed to open temp image file",
"temp_file_path", tmpFilename,
"image_url", img.URL,
)
}
_, err = io.Copy(file, img.File)
if err != nil {
_ = file.Close()
return nil, errs.Wrapw(err, "failed to download image to temp file",
"temp_file_path", tmpFilename,
"image_url", img.URL,
)
}
return &tempFile{
file: file,
filename: tmpFilename,
}, err
}

View file

@ -8,7 +8,6 @@ import (
"github.com/alecthomas/units" "github.com/alecthomas/units"
"github.com/tigorlazuardi/redmage/api/bmessage" "github.com/tigorlazuardi/redmage/api/bmessage"
"github.com/tigorlazuardi/redmage/pkg/errs" "github.com/tigorlazuardi/redmage/pkg/errs"
"golang.org/x/sync/errgroup"
) )
type DownloadStatusBroadcaster interface { type DownloadStatusBroadcaster interface {
@ -20,38 +19,42 @@ type NullDownloadStatusBroadcaster struct{}
func (NullDownloadStatusBroadcaster) Broadcast(bmessage.ImageDownloadMessage) {} func (NullDownloadStatusBroadcaster) Broadcast(bmessage.ImageDownloadMessage) {}
type PostImage struct { type PostImage struct {
ImageURL string URL string
ImageFile io.Reader File io.ReadCloser
ThumbnailURL string
ThumbnailFile io.Reader
} }
func (po *PostImage) Read(p []byte) (n int, err error) {
return po.File.Read(p)
}
func (po *PostImage) Close() error {
return po.File.Close()
}
// DownloadImage downloades the image.
//
// If downloading image or thumbnail fails
func (reddit *Reddit) DownloadImage(ctx context.Context, post Post, broadcaster DownloadStatusBroadcaster) (image PostImage, err error) { func (reddit *Reddit) DownloadImage(ctx context.Context, post Post, broadcaster DownloadStatusBroadcaster) (image PostImage, err error) {
imageUrl, thumbnailUrl := post.GetImageURL(), post.GetThumbnailURL() imageUrl := post.GetImageURL()
image.ImageURL = imageUrl image.URL = imageUrl
image.ThumbnailURL = thumbnailUrl
group, groupCtx := errgroup.WithContext(ctx) image.File, err = reddit.downloadImage(ctx, post, bmessage.KindImage, broadcaster)
group.Go(func() error {
var err error
image.ImageFile, err = reddit.downloadImage(groupCtx, post, bmessage.KindImage, broadcaster)
return err
})
group.Go(func() error {
var err error
image.ThumbnailFile, err = reddit.downloadImage(groupCtx, post, bmessage.KindThumbnail, broadcaster)
return err
})
err = group.Wait()
return image, err return image, err
} }
func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessage.ImageKind, broadcaster DownloadStatusBroadcaster) (io.Reader, error) { func (reddit *Reddit) DownloadThumbnail(ctx context.Context, post Post, broadcaster DownloadStatusBroadcaster) (image PostImage, err error) {
imageUrl := post.GetThumbnailURL()
image.URL = imageUrl
image.File, err = reddit.downloadImage(ctx, post, bmessage.KindThumbnail, broadcaster)
return image, err
}
func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessage.ImageKind, broadcaster DownloadStatusBroadcaster) (io.ReadCloser, error) {
var ( var (
url string url string
height int height int64
width int width int64
) )
if kind == bmessage.KindImage { if kind == bmessage.KindImage {
url = post.GetImageURL() url = post.GetImageURL()
@ -75,15 +78,23 @@ func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessag
if metricSpeed == 0 { if metricSpeed == 0 {
metricSpeed = 10 * units.KB metricSpeed = 10 * units.KB
} }
idr := &ImageDownloadReader{ metadata := bmessage.ImageMetadata{
OnProgress: func(downloaded int64, contentLength int64, err error) {
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
Metadata: bmessage.ImageMetadata{
URL: url, URL: url,
Height: height, Height: height,
Width: width, Width: width,
Kind: kind, Kind: kind,
}, }
idr := &ImageDownloadReader{
OnProgress: func(downloaded int64, contentLength int64, err error) {
var event bmessage.DownloadEvent
if err != nil {
event = bmessage.DownloadError
} else {
event = bmessage.DownloadProgress
}
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
Event: event,
Metadata: metadata,
ContentLength: units.MetricBytes(resp.ContentLength), ContentLength: units.MetricBytes(resp.ContentLength),
Downloaded: units.MetricBytes(downloaded), Downloaded: units.MetricBytes(downloaded),
Subreddit: post.GetSubreddit(), Subreddit: post.GetSubreddit(),
@ -92,6 +103,18 @@ func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessag
Error: err, Error: err,
}) })
}, },
OnClose: func(downloaded, contentLength int64, closeErr error) {
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
Event: bmessage.DownloadEnd,
Metadata: metadata,
ContentLength: units.MetricBytes(resp.ContentLength),
Downloaded: units.MetricBytes(downloaded),
Subreddit: post.GetSubreddit(),
PostURL: post.GetPermalink(),
PostID: post.GetID(),
Error: closeErr,
})
},
IdleTimeout: reddit.Config.Duration("download.timeout.idle"), IdleTimeout: reddit.Config.Duration("download.timeout.idle"),
IdleSpeedThreshold: metricSpeed, IdleSpeedThreshold: metricSpeed,
} }
@ -100,6 +123,18 @@ func (reddit *Reddit) downloadImage(ctx context.Context, post Post, kind bmessag
reader, writer := io.Pipe() reader, writer := io.Pipe()
go func() { go func() {
defer resp.Body.Close() defer resp.Body.Close()
broadcaster.Broadcast(bmessage.ImageDownloadMessage{
Event: bmessage.DownloadStart,
Metadata: bmessage.ImageMetadata{
URL: url,
Height: height,
Width: width,
Kind: kind,
},
Subreddit: post.GetSubreddit(),
PostURL: post.GetPermalink(),
PostID: post.GetID(),
})
_, err := io.Copy(writer, resp.Body) _, err := io.Copy(writer, resp.Body)
_ = writer.CloseWithError(err) _ = writer.CloseWithError(err)
}() }()

View file

@ -53,7 +53,7 @@ func (reddit *Reddit) GetPosts(ctx context.Context, params GetPostsParam) (posts
return posts, errs.Fail("reddit: unexpected status code when executing GetPosts", return posts, errs.Fail("reddit: unexpected status code when executing GetPosts",
slog.Group("request", "url", url, "params", params), slog.Group("request", "url", url, "params", params),
slog.Group("response", "status_code", res.StatusCode, "body", formatLogBody(res, body)), slog.Group("response", "status_code", res.StatusCode, "body", formatLogBody(res, body)),
) ).Code(res.StatusCode)
} }
err = json.NewDecoder(res.Body).Decode(&posts) err = json.NewDecoder(res.Body).Decode(&posts)

View file

@ -15,7 +15,7 @@ var ErrIdleTimeoutReached = errors.New("download idle timeout reached")
type ImageDownloadReader struct { type ImageDownloadReader struct {
OnProgress func(downloaded int64, contentLength int64, err error) OnProgress func(downloaded int64, contentLength int64, err error)
OnClose func(closeErr error) OnClose func(downloaded int64, contentLength int64, closeErr error)
IdleTimeout time.Duration IdleTimeout time.Duration
IdleSpeedThreshold units.MetricBytes IdleSpeedThreshold units.MetricBytes
@ -107,7 +107,7 @@ func (idr *ImageDownloadReader) Close() error {
idr.exit <- struct{}{} idr.exit <- struct{}{}
err := idr.reader.Close() err := idr.reader.Close()
if idr.OnClose != nil { if idr.OnClose != nil {
idr.OnClose(err) idr.OnClose(idr.downloaded.Load(), idr.contentLength, err)
} }
return err return err
} }

View file

@ -1,5 +1,15 @@
package reddit package reddit
import (
"fmt"
"net/url"
"path"
"strings"
"github.com/tigorlazuardi/redmage/config"
"github.com/tigorlazuardi/redmage/db/queries"
)
type Listing struct { type Listing struct {
Kind string `json:"kind"` Kind string `json:"kind"`
Data Data `json:"data"` Data Data `json:"data"`
@ -15,8 +25,8 @@ type (
Gildings struct{} Gildings struct{}
Source struct { Source struct {
URL string `json:"url"` URL string `json:"url"`
Width int `json:"width"` Width int64 `json:"width"`
Height int `json:"height"` Height int64 `json:"height"`
} }
) )
@ -86,7 +96,7 @@ type PostData struct {
Pwls int `json:"pwls"` Pwls int `json:"pwls"`
LinkFlairCSSClass string `json:"link_flair_css_class"` LinkFlairCSSClass string `json:"link_flair_css_class"`
Downs int `json:"downs"` Downs int `json:"downs"`
ThumbnailHeight int `json:"thumbnail_height"` ThumbnailHeight int64 `json:"thumbnail_height"`
TopAwardedType any `json:"top_awarded_type"` TopAwardedType any `json:"top_awarded_type"`
HideScore bool `json:"hide_score"` HideScore bool `json:"hide_score"`
MediaMetadata map[string]MediaMetadata `json:"media_metadata"` MediaMetadata map[string]MediaMetadata `json:"media_metadata"`
@ -98,7 +108,7 @@ type PostData struct {
Ups int `json:"ups"` Ups int `json:"ups"`
Domain string `json:"domain"` Domain string `json:"domain"`
MediaEmbed MediaEmbed `json:"media_embed"` MediaEmbed MediaEmbed `json:"media_embed"`
ThumbnailWidth int `json:"thumbnail_width"` ThumbnailWidth int64 `json:"thumbnail_width"`
AuthorFlairTemplateID string `json:"author_flair_template_id"` AuthorFlairTemplateID string `json:"author_flair_template_id"`
IsOriginalContent bool `json:"is_original_content"` IsOriginalContent bool `json:"is_original_content"`
UserReports []any `json:"user_reports"` UserReports []any `json:"user_reports"`
@ -189,6 +199,10 @@ type Post struct {
Data PostData `json:"data,omitempty"` Data PostData `json:"data,omitempty"`
} }
func (post *Post) IsNSFW() bool {
return post.Data.Over18
}
func (post *Post) IsImagePost() bool { func (post *Post) IsImagePost() bool {
return post.Data.PostHint == "image" return post.Data.PostHint == "image"
} }
@ -197,7 +211,44 @@ func (post *Post) GetImageURL() string {
return post.Data.URL return post.Data.URL
} }
func (post *Post) GetImageSize() (width, height int) { func (post *Post) GetImageAspectRatio() float64 {
width, height := post.GetImageSize()
if height == 0 {
return 0
}
return float64(width) / float64(height)
}
func (post *Post) GetImageTargetPath(cfg *config.Config, device queries.Device) string {
baseDownloadDir := cfg.String("download.directory")
return path.Join(baseDownloadDir, device.Name, post.GetSubreddit(), post.GetImageFilename())
}
func (post *Post) GetWindowsWallpaperImageTargetPath(cfg *config.Config, device queries.Device) string {
baseDownloadDir := cfg.String("download.directory")
filename := fmt.Sprintf("%s_%s", post.GetSubreddit(), post.GetImageFilename())
return path.Join(baseDownloadDir, device.Name, filename)
}
func (post *Post) GetThumbnailTargetPath(cfg *config.Config) string {
baseDownloadDir := cfg.String("download.directory")
return path.Join(baseDownloadDir, "_thumbnails", post.GetSubreddit(), post.GetImageFilename())
}
func (post *Post) GetImageFilename() string {
if !post.IsImagePost() {
return ""
}
link := post.GetImageURL()
u, _ := url.Parse(link)
if u == nil {
return ""
}
split := strings.Split(u.Path, "/")
return split[len(split)-1]
}
func (post *Post) GetImageSize() (width, height int64) {
if len(post.Data.Preview.Images) == 0 { if len(post.Data.Preview.Images) == 0 {
return 0, 0 return 0, 0
} }
@ -209,7 +260,7 @@ func (post *Post) GetThumbnailURL() string {
return post.Data.Thumbnail return post.Data.Thumbnail
} }
func (post *Post) GetThumbnailSize() (width, height int) { func (post *Post) GetThumbnailSize() (width, height int64) {
return post.Data.ThumbnailWidth, post.Data.ThumbnailHeight return post.Data.ThumbnailWidth, post.Data.ThumbnailHeight
} }

View file

@ -2,10 +2,12 @@ package cli
import ( import (
"io/fs" "io/fs"
"net/http"
"os" "os"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/tigorlazuardi/redmage/api" "github.com/tigorlazuardi/redmage/api"
"github.com/tigorlazuardi/redmage/api/reddit"
"github.com/tigorlazuardi/redmage/db" "github.com/tigorlazuardi/redmage/db"
"github.com/tigorlazuardi/redmage/db/queries" "github.com/tigorlazuardi/redmage/db/queries"
"github.com/tigorlazuardi/redmage/pkg/log" "github.com/tigorlazuardi/redmage/pkg/log"
@ -35,7 +37,17 @@ var serveCmd = &cobra.Command{
queries := queries.New(db) queries := queries.New(db)
api := api.New(queries, db, cfg) red := &reddit.Reddit{
Client: http.DefaultClient,
Config: cfg,
}
api := api.New(api.Dependencies{
Queries: queries,
DB: db,
Config: cfg,
Reddit: red,
})
server := server.New(cfg, api, PublicDir) server := server.New(cfg, api, PublicDir)

View file

@ -15,7 +15,8 @@ var DefaultConfig = map[string]any{
"db.string": "data.db", "db.string": "data.db",
"db.automigrate": true, "db.automigrate": true,
"download.concurrency": 5, "download.concurrency.images": 5,
"download.concurrency.subreddits": 3,
"download.directory": "", "download.directory": "",
"download.timeout.headers": "10s", "download.timeout.headers": "10s",
"download.timeout.idle": "5s", "download.timeout.idle": "5s",

View file

@ -11,6 +11,7 @@ CREATE TABLE devices(
max_x INTEGER NOT NULL DEFAULT 0, max_x INTEGER NOT NULL DEFAULT 0,
max_y INTEGER NOT NULL DEFAULT 0, max_y INTEGER NOT NULL DEFAULT 0,
nsfw INTEGER NOT NULL DEFAULT 0, nsfw INTEGER NOT NULL DEFAULT 0,
windows_wallpaper_mode INTEGER NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
); );

2
go.mod
View file

@ -34,6 +34,7 @@ require (
github.com/samber/lo v1.38.1 // indirect github.com/samber/lo v1.38.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect
golang.org/x/net v0.23.0 // indirect golang.org/x/net v0.23.0 // indirect
golang.org/x/text v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
@ -44,6 +45,7 @@ require (
require ( require (
github.com/XSAM/otelsql v0.29.0 github.com/XSAM/otelsql v0.29.0
github.com/disintegration/imaging v1.6.2
github.com/felixge/httpsnoop v1.0.4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/logr v1.4.1 // indirect

5
go.sum
View file

@ -33,6 +33,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
@ -248,6 +250,8 @@ golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE= golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE=
golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
@ -264,6 +268,7 @@ golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=

31
pkg/telemetry/status.go Normal file
View file

@ -0,0 +1,31 @@
package telemetry
import (
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// EndWithStatus ends the span with the status of the error if not nil
// otherwise it will set the status to OK.
//
// This function should be used for ending spans, not for starting spans
// or spans that will have children, to avoid duplicate error recordings.
//
// Do not defer this function directly since err might be nil at the
// start of defer call. Instead it should be wrapped in a function to
// capture the error correctly.
//
// Example:
//
// var err error
// ctx, span := tracer.Start(ctx, "my-operation")
// defer func() { telemetry.EndWithStatus(span, err) }()
func EndWithStatus(span trace.Span, err error) {
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
} else {
span.SetStatus(codes.Ok, "")
}
span.End()
}