website-feeds

  1package server
  2
  3// TODO: Currently, there's no content in the feed entries.  It would be great
  4// to scrape content at least for text and image posts to eliminate a
  5// click-through.
  6
  7import (
  8	"context"
  9	"errors"
 10	"fmt"
 11	"log/slog"
 12	"math/rand/v2"
 13	"net/http"
 14	"net/netip"
 15	"strconv"
 16	"strings"
 17	"time"
 18
 19	"website-feeds/hackernews"
 20	"website-feeds/lobsters"
 21	"website-feeds/model"
 22	"website-feeds/reddit"
 23	"website-feeds/store"
 24
 25	"github.com/gorilla/feeds"
 26	"github.com/jackc/pgx/v5"
 27)
 28
 29// TODO: Create a fake Fetcher for testing purposes.  I think as part of this
 30// we're going to need to move all feed knowledge out of the server into a new
 31// package.
 32type Fetcher interface {
 33	FetchPosts(ctx context.Context, numPosts int) ([]model.Post, error)
 34	DisplayName(ctx context.Context) (string, error)
 35	URL() string
 36	CommentURL(post model.Post) string
 37}
 38
 39var (
 40	_ Fetcher = (*reddit.Client)(nil)
 41	_ Fetcher = (*hackernews.Client)(nil)
 42	_ Fetcher = (*lobsters.Client)(nil)
 43)
 44
 45type (
 46	requestIDKey       struct{}
 47	requestHostKey     struct{}
 48	requestSettingsKey struct{}
 49)
 50
 51func handleRoot() http.Handler {
 52	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 53		http.ServeFileFS(w, r, static, "static/root.html")
 54	})
 55}
 56
 57func handleGetSettings() http.Handler {
 58	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 59		rID := r.Context().Value(requestIDKey{}).(int64)
 60		rSettings := r.Context().Value(requestSettingsKey{}).(*model.Settings)
 61
 62		w.Header().Set("Content-Type", "text/html")
 63		if err := RenderSettings(w, settingsTemplateArgsFromSettings(rSettings)); err != nil {
 64			slog.Error(
 65				"Failed to render /settings",
 66				slog.Any("id", rID),
 67				slog.Any("err", err),
 68			)
 69		}
 70	})
 71}
 72
 73func handlePostSettings(s store.Store) http.Handler {
 74	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 75		rID := r.Context().Value(requestIDKey{}).(int64)
 76
 77		redditClientIdS := r.FormValue("reddit-client-id")
 78		redditClientSecretS := r.FormValue("reddit-client-secret")
 79		defaultNumPostsS := r.FormValue("default-num-posts")
 80		maxNumPostsS := r.FormValue("max-num-posts")
 81		siteFetchWaitS := r.FormValue("site-fetch-wait")
 82
 83		slog.Info(
 84			"POST /settings",
 85			slog.Any("id", rID),
 86			slog.Any("reddit-client-id", redditClientIdS),
 87			slog.Any("reddit-client-secret", redditClientSecretS),
 88			slog.Any("default-num-posts", defaultNumPostsS),
 89			slog.Any("max-num-posts", maxNumPostsS),
 90			slog.Any("site-fetch-wait", siteFetchWaitS),
 91		)
 92
 93		var redditClientId *string
 94		if redditClientIdS != "" {
 95			redditClientId = &redditClientIdS
 96		}
 97
 98		var redditClientSecret *string
 99		if redditClientSecretS != "" {
100			redditClientSecret = &redditClientSecretS
101		}
102
103		if (redditClientId == nil) != (redditClientSecret == nil) {
104			http.Error(
105				w,
106				"Must set either both of or neither of Reddit Client ID and Reddit Client Secret",
107				http.StatusBadRequest,
108			)
109			return
110		}
111
112		defaultNumPosts, err := strconv.Atoi(defaultNumPostsS)
113		if err != nil {
114			http.Error(
115				w,
116				"Default num posts must be a number",
117				http.StatusBadRequest,
118			)
119			return
120		}
121
122		maxNumPosts, err := strconv.Atoi(maxNumPostsS)
123		if err != nil {
124			http.Error(
125				w,
126				"Max num posts must be a number",
127				http.StatusBadRequest,
128			)
129			return
130		}
131
132		siteFetchWaitN, err := strconv.Atoi(siteFetchWaitS)
133		if err != nil {
134			http.Error(
135				w,
136				"Site fetch wait must be a number",
137				http.StatusBadRequest,
138			)
139			return
140		}
141
142		var reddit *model.RedditSettings
143		if redditClientId != nil && redditClientSecret != nil {
144			reddit = &model.RedditSettings{
145				ClientID:     *redditClientId,
146				ClientSecret: *redditClientSecret,
147			}
148		}
149
150		siteFetchWait := time.Duration(siteFetchWaitN) * time.Hour
151
152		if err := s.UpdateSettings(r.Context(), &model.Settings{
153			DefaultNumPosts: defaultNumPosts,
154			MaxNumPosts:     maxNumPosts,
155			SiteFetchWait:   siteFetchWait,
156			Reddit:          reddit,
157		}); err != nil {
158			slog.Error(
159				"Failed to update settings",
160				slog.Any("id", rID),
161				slog.Any("redditClientId", redditClientId),
162				slog.Any("redditClientSecret", redditClientSecret),
163				slog.Any("defaultNumPosts", defaultNumPosts),
164				slog.Any("maxNumPosts", maxNumPosts),
165				slog.Any("siteFetchWait", siteFetchWait),
166				slog.Any("err", err),
167			)
168			http.Error(
169				w,
170				"Internal Server Error",
171				http.StatusInternalServerError,
172			)
173			return
174		}
175
176		http.Redirect(w, r, "/settings", http.StatusSeeOther)
177	})
178}
179
180func handleFeed(s store.Store) http.Handler {
181	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
182		rID := r.Context().Value(requestIDKey{}).(int64)
183		rSettings := r.Context().Value(requestSettingsKey{}).(*model.Settings)
184		rHost := r.Context().Value(requestHostKey{}).(netip.AddrPort)
185
186		siteS := strings.TrimSpace(r.PathValue("site"))
187		numPostsS := strings.TrimSpace(r.URL.Query().Get("num"))
188
189		numPosts := rSettings.DefaultNumPosts
190		defaultNumPosts := true
191		if numPostsS != "" {
192			var err error
193			numPosts, err = strconv.Atoi(numPostsS)
194			defaultNumPosts = false
195			if err != nil {
196				http.Error(
197					w,
198					"num parameter was not a number",
199					http.StatusBadRequest,
200				)
201				return
202			}
203			if numPosts > rSettings.MaxNumPosts {
204				http.Error(
205					w,
206					fmt.Sprintf(
207						"num parameter was bigger than maximum (%d)",
208						rSettings.MaxNumPosts,
209					),
210					http.StatusBadRequest,
211				)
212				return
213			}
214		}
215
216		var f Fetcher
217		if siteS == "hackernews" {
218			f = hackernews.New()
219		} else if siteS == "lobsters" {
220			f = lobsters.New()
221		} else if subreddit, found := strings.CutPrefix(siteS, "reddit:"); found {
222			if rSettings.Reddit == nil {
223				slog.Warn(
224					"Reddit settings were nil",
225					slog.Any("id", rID),
226					slog.Any("siteS", siteS),
227				)
228				http.Error(w, "Reddit settings are not set", http.StatusBadRequest)
229				return
230			}
231
232			f = reddit.New(rSettings.Reddit.ClientID, rSettings.Reddit.ClientSecret, subreddit)
233		} else {
234			slog.Warn(
235				"Unrecognized site",
236				slog.Any("id", rID),
237				slog.Any("siteS", siteS),
238			)
239			http.Error(w, "Site not recognized", http.StatusBadRequest)
240			return
241		}
242
243		slog.Info(
244			"Getting site from store",
245			slog.Any("id", rID),
246			slog.Any("site", siteS),
247			slog.Any("numPosts", numPosts),
248		)
249		site, err := s.SiteByName(r.Context(), siteS)
250		if err != nil {
251			if errors.Is(err, pgx.ErrNoRows) {
252				slog.Info(
253					"Site not in store. Fetching name",
254					slog.Any("id", rID),
255					slog.Any("site", siteS),
256				)
257				displayName, err := f.DisplayName(r.Context())
258				if err != nil {
259					slog.Error(
260						"Couldn't fetch display name",
261						slog.Any("id", rID),
262						slog.Any("site", siteS),
263						slog.Any("err", err),
264					)
265					http.Error(
266						w,
267						"Internal Server Error",
268						http.StatusInternalServerError,
269					)
270					return
271				}
272				slog.Info(
273					"Creating site",
274					slog.Any("id", rID),
275					slog.Any("site", siteS),
276					slog.Any("displayName", displayName),
277				)
278				site, err = s.CreateSite(r.Context(), siteS, displayName)
279				if err != nil {
280					slog.Error(
281						"Couldn't create site",
282						slog.Any("id", rID),
283						slog.Any("site", siteS),
284						slog.Any("displayName", displayName),
285						slog.Any("err", err),
286					)
287					http.Error(
288						w,
289						"Internal Server Error",
290						http.StatusInternalServerError,
291					)
292					return
293				}
294			} else {
295				slog.Error(
296					"Couldn't fetch site from store",
297					slog.Any("id", rID),
298					slog.Any("site", siteS),
299					slog.Any("err", err),
300				)
301				http.Error(w, "Internal Server Error", http.StatusInternalServerError)
302				return
303			}
304		}
305
306		now := time.Now()
307		if err := s.WriteRequest(r.Context(), &model.Request{
308			Host:            netip.AddrPort(rHost),
309			SiteId:          site.ID,
310			NumPosts:        numPosts,
311			DefaultNumPosts: defaultNumPosts,
312			Timestamp:       now,
313		}); err != nil {
314			slog.Error(
315				"Failed to write request to store",
316				slog.Any("id", rID),
317				slog.Any("site", site.Name),
318				slog.Any("numPosts", numPosts),
319				slog.Any("err", err),
320			)
321			http.Error(
322				w,
323				"Internal Server Error",
324				http.StatusInternalServerError,
325			)
326			return
327		}
328
329		numFetchPosts, err := s.NumFetchPostsForSite(r.Context(), site.ID, now)
330		if err != nil {
331			slog.Info(
332				"Couldn't get numFetchPosts for site",
333				slog.Any("id", rID),
334				slog.Any("site", site.Name),
335				slog.Any("err", err),
336			)
337			http.Error(
338				w,
339				"Internal Server Error",
340				http.StatusInternalServerError,
341			)
342			return
343		}
344
345		if site.LastFetched == nil ||
346			now.Sub(site.LastFetched.Time) > rSettings.SiteFetchWait ||
347			numFetchPosts > site.LastFetched.Num {
348			var lastFetchedTime *time.Time
349			var lastFetchedNum *int
350			if site.LastFetched != nil {
351				lastFetchedTime = &site.LastFetched.Time
352				lastFetchedNum = &site.LastFetched.Num
353			}
354
355			slog.Info(
356				"Fetching posts from site",
357				slog.Any("id", rID),
358				slog.Any("site", site.Name),
359				slog.Any("lastFetchedTime", lastFetchedTime),
360				slog.Any("lastFetchedNum", lastFetchedNum),
361				slog.Any("numFetchPosts", numFetchPosts),
362			)
363
364			posts, err := f.FetchPosts(r.Context(), numFetchPosts)
365			if err != nil {
366				slog.Info(
367					"Couldn't fetch posts from site",
368					slog.Any("id", rID),
369					slog.Any("site", site.Name),
370					slog.Any("err", err),
371				)
372				http.Error(
373					w,
374					"Internal Server Error",
375					http.StatusInternalServerError,
376				)
377				return
378			}
379
380			slog.Info(
381				"Writing posts for site to store",
382				slog.Any("id", rID),
383				slog.Any("site", site.Name),
384			)
385			err = s.WriteSitePosts(
386				r.Context(),
387				site.ID,
388				posts,
389				model.SiteLastFetched{
390					Time: now,
391					Num:  numFetchPosts,
392				},
393			)
394			if err != nil {
395				slog.Error(
396					"Couldn't write posts to store",
397					slog.Any("id", rID),
398					slog.Any("site", site.Name),
399					slog.Any("err", err),
400				)
401				http.Error(
402					w,
403					"Internal Server Error",
404					http.StatusInternalServerError,
405				)
406				return
407			}
408		} else {
409			slog.Info(
410				"Not fetching posts for site",
411				slog.Any("id", rID),
412				slog.Any("site", site.Name),
413				slog.Any("lastFetchedTime", site.LastFetched.Time),
414				slog.Any("lastFetchedNum", site.LastFetched.Num),
415			)
416		}
417
418		slog.Info(
419			"Getting top posts from store",
420			slog.Any("id", rID),
421			slog.Any("site", site.Name),
422			slog.Any("numPosts", numPosts),
423		)
424		posts, err := s.TopPostsForSite(r.Context(), site.ID, now, numPosts)
425		if err != nil {
426			slog.Error(
427				"Couldn't get top posts from store",
428				slog.Any("id", rID),
429				slog.Any("site", site.Name),
430				slog.Any("err", err),
431			)
432			http.Error(
433				w,
434				"Internal Server Error",
435				http.StatusInternalServerError,
436			)
437			return
438		}
439
440		feed := &feeds.Feed{
441			Title:       site.DisplayName,
442			Link:        &feeds.Link{Href: f.URL()},
443			Description: fmt.Sprintf("Top posts for %s", site.DisplayName),
444		}
445
446		for _, post := range posts {
447			commentURL := f.CommentURL(post)
448			feed.Add(&feeds.Item{
449				Id:      fmt.Sprintf("%s:%s", site.Name, post.SiteUniqueID),
450				Title:   post.Title,
451				Link:    &feeds.Link{Href: commentURL},
452				Created: post.Created,
453			})
454		}
455
456		slog.Info(
457			"Writing feed",
458			slog.Any("id", rID),
459			slog.Any("site", site.Name),
460		)
461		w.Header().Set("Content-Type", "application/atom+xml")
462		if err := feed.WriteAtom(w); err != nil {
463			slog.Error(
464				"Couldn't write atom to client",
465				slog.Any("id", rID),
466				slog.Any("err", err),
467			)
468		}
469	})
470}
471
472func idMiddleware(next http.Handler) http.Handler {
473	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
474		id := rand.Int64()
475
476		newR := r.WithContext(
477			context.WithValue(r.Context(), requestIDKey{}, id),
478		)
479
480		next.ServeHTTP(w, newR)
481	})
482}
483
484func hostMiddleware(next http.Handler) http.Handler {
485	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
486		rID := r.Context().Value(requestIDKey{}).(int64)
487
488		host, err := netip.ParseAddrPort(r.RemoteAddr)
489		if err != nil {
490			slog.Error(
491				"Unable to parse remote address",
492				slog.Any("id", rID),
493				slog.Any("err", err),
494			)
495			http.Error(
496				w,
497				"Internal Server Error",
498				http.StatusInternalServerError,
499			)
500			return
501
502		}
503
504		newR := r.WithContext(
505			context.WithValue(r.Context(), requestHostKey{}, host),
506		)
507
508		next.ServeHTTP(w, newR)
509	})
510}
511
512func logMiddleware(next http.Handler) http.Handler {
513	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
514		rID := r.Context().Value(requestIDKey{}).(int64)
515		rHost := r.Context().Value(requestHostKey{}).(netip.AddrPort)
516
517		slog.Info(
518			"New request",
519			slog.Any("id", rID),
520			slog.Any("peer", rHost.String()),
521			slog.Any("url", r.URL),
522		)
523
524		next.ServeHTTP(w, r)
525	})
526}
527
528func getSettingsMiddlewareFunc(
529	s store.Store,
530) func(http.Handler) http.Handler {
531	return func(next http.Handler) http.Handler {
532		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
533			rID := r.Context().Value(requestIDKey{}).(int64)
534
535			settings, err := s.Settings(r.Context())
536			if err != nil {
537				slog.Error(
538					"Failed to fetch settings",
539					slog.Any("id", rID),
540					slog.Any("err", err),
541				)
542				http.Error(
543					w,
544					"Internal Server Error",
545					http.StatusInternalServerError,
546				)
547				return
548			}
549
550			newR := r.WithContext(
551				context.WithValue(r.Context(), requestSettingsKey{}, settings),
552			)
553
554			next.ServeHTTP(w, newR)
555		})
556	}
557}
558
559func newHandler(
560	s store.Store,
561) http.Handler {
562	mux := http.NewServeMux()
563
564	settingsMiddleware := getSettingsMiddlewareFunc(s)
565
566	mux.Handle("GET /", handleRoot())
567	mux.Handle("GET /feed/{site}", settingsMiddleware(handleFeed(s)))
568	mux.Handle("GET /settings", settingsMiddleware(handleGetSettings()))
569	mux.Handle("POST /settings", handlePostSettings(s))
570	mux.Handle("GET /static/", http.FileServerFS(static))
571
572	return idMiddleware(hostMiddleware(logMiddleware(mux)))
573}
574
575func Run(
576	ctx context.Context,
577	cancel context.CancelFunc,
578	s store.Store,
579	bindAddr string,
580) {
581	handler := newHandler(s)
582	srv := http.Server{
583		Handler: handler,
584		Addr:    bindAddr,
585	}
586
587	go func() {
588		slog.Info("Starting to serve", slog.Any("addr", bindAddr))
589		if err := srv.ListenAndServe(); err != nil &&
590			err != http.ErrServerClosed {
591			slog.Info("Failed to serve", slog.Any("err", err))
592			cancel()
593		}
594	}()
595
596	<-ctx.Done()
597
598	slog.Info("Beginning shutdown")
599
600	shutdownCtx := context.Background()
601	shutdownCtx, cancel = context.WithTimeout(shutdownCtx, 10*time.Second)
602	defer cancel()
603	if err := srv.Shutdown(shutdownCtx); err != nil {
604		slog.Info("error shutting down", slog.Any("err", err))
605	}
606}