Files
pages/internal/server/feeds_fetch.go
T
Anya Tokugawa bfed3d20f5
Build and publish container images / images (push) Failing after 6m7s
feat(feeds): cache FreshRSS subscriptions and category navigation
2026-11-08 04:48:55 +09:00

537 lines
15 KiB
Go

package server
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"html"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode/utf8"
"github.com/microcosm-cc/bluemonday"
xhtml "golang.org/x/net/html"
"golang.org/x/net/html/charset"
"pageswiki/internal/store"
)
var errFeedInvalid = errors.New("invalid_feed")
var feedTextPolicy = bluemonday.StrictPolicy()
var feedHTMLPolicy = func() *bluemonday.Policy {
p := bluemonday.NewPolicy()
p.AllowElements("p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "b", "em", "i", "u", "s", "del", "ins", "sub", "sup", "mark", "small", "blockquote", "pre", "code", "kbd", "samp", "ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td", "a")
p.AllowAttrs("href", "target", "rel").OnElements("a")
p.AllowAttrs("colspan", "rowspan", "scope").OnElements("th", "td")
p.AllowStandardURLs()
p.AllowRelativeURLs(false)
return p
}()
// Keep article structure without granting fetched HTML script, style, form,
// embedding or automatic network capabilities. Wiki capture still uses Body.
func feedHTML(raw, base string) string {
if raw == "" || len(raw) > 128<<10 {
return ""
}
z := xhtml.NewTokenizer(strings.NewReader(raw))
var out strings.Builder
markup := false
for {
typeOf := z.Next()
if typeOf == xhtml.ErrorToken {
if z.Err() != io.EOF {
return ""
}
break
}
if typeOf == xhtml.StartTagToken || typeOf == xhtml.SelfClosingTagToken {
markup = true
tok := z.Token()
if tok.Data == "a" {
attrs := []xhtml.Attribute{}
for _, attr := range tok.Attr {
if attr.Key == "href" {
if link := feedLink(base, attr.Val); link != "" {
attrs = append(attrs, xhtml.Attribute{Key: "href", Val: link})
}
}
}
tok.Attr = append(attrs, xhtml.Attribute{Key: "target", Val: "_blank"}, xhtml.Attribute{Key: "rel", Val: "noopener noreferrer"})
}
out.WriteString(tok.String())
} else {
out.Write(z.Raw())
}
}
if !markup {
return ""
}
clean := feedHTMLPolicy.Sanitize(out.String())
if len(clean) > 128<<10 {
return ""
}
return clean
}
func feedText(raw string, limit int) string {
// Plain text for fallback and inert Wiki/task capture.
for _, tag := range []string{"</p>", "</div>", "<br>", "<br/>", "<br />", "</li>"} {
raw = strings.ReplaceAll(raw, tag, tag+"\n")
}
text := strings.TrimSpace(html.UnescapeString(feedTextPolicy.Sanitize(raw)))
if len(text) > limit {
text = text[:limit-16]
for !utf8.ValidString(text) {
text = text[:len(text)-1]
}
text += "\n[… truncated]"
}
return text
}
func feedLink(base, raw string) string {
if strings.TrimSpace(raw) == "" || len(raw) > 4096 {
return ""
}
u, e := url.Parse(strings.TrimSpace(raw))
if e != nil {
return ""
}
b, e := url.Parse(base)
if e != nil {
return ""
}
u = b.ResolveReference(u)
if (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
return ""
}
result := u.String()
if len(result) > 4096 {
return ""
}
return result
}
func feedDate(v string) int64 {
for _, layout := range []string{time.RFC3339Nano, time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, "2006-01-02T15:04:05-0700"} {
if t, e := time.Parse(layout, v); e == nil && t.UnixMilli() > 0 && t.Before(time.Now().AddDate(1, 0, 0)) {
return t.UnixMilli()
}
}
return 0
}
type rssEntry struct {
ID string `xml:"guid"`
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
Content string `xml:"encoded"`
Date string `xml:"pubDate"`
Category string `xml:"category"`
}
type atomText struct {
Type string `xml:"type,attr"`
Text string `xml:",chardata"`
Inner string `xml:",innerxml"`
}
func (v atomText) body() (string, bool) {
if v.Type == "xhtml" {
return v.Inner, true
}
return v.Text, v.Type == "html"
}
type atomEntry struct {
ID string `xml:"id"`
Title string `xml:"title"`
Links []struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
} `xml:"link"`
Summary atomText `xml:"summary"`
Content atomText `xml:"content"`
Published string `xml:"published"`
Updated string `xml:"updated"`
Category struct {
Term string `xml:"term,attr"`
} `xml:"category"`
}
func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
if len(raw) > 4<<20 {
return nil, errors.New("limit")
}
d := xml.NewDecoder(bytes.NewReader(raw))
d.CharsetReader = charset.NewReaderLabel
root := ""
title := ""
depth := 0
out := []store.FeedItem{}
seen := map[string]bool{}
for {
tok, e := d.Token()
if e == io.EOF {
break
}
if e != nil {
return nil, errFeedInvalid
}
switch t := tok.(type) {
case xml.Directive:
return nil, errFeedInvalid // DTD/entities have no role in a feed.
case xml.StartElement:
depth++
if depth == 1 && root != "" {
return nil, errFeedInvalid
}
if depth > 64 {
return nil, errFeedInvalid
}
if root == "" {
root = t.Name.Local
if root != "rss" && root != "feed" {
return nil, errFeedInvalid
}
}
var item store.FeedItem
isItem := false
htmlBody := false
if root == "rss" && t.Name.Local == "item" && depth == 3 {
var v rssEntry
if d.DecodeElement(&v, &t) != nil {
return nil, errFeedInvalid
}
depth--
body := v.Content
if body == "" {
body = v.Description
}
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category}
isItem = true
htmlBody = true
} else if root == "feed" && t.Name.Local == "entry" && depth == 2 {
var v atomEntry
if d.DecodeElement(&v, &t) != nil {
return nil, errFeedInvalid
}
depth--
body, isHTML := v.Content.body()
if body == "" {
body, isHTML = v.Summary.body()
}
htmlBody = isHTML
link := ""
for _, l := range v.Links {
if l.Rel == "" || l.Rel == "alternate" {
link = feedLink(base, l.Href)
break
}
}
date := v.Published
if date == "" {
date = v.Updated
}
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: link, Body: body, PublishedMS: feedDate(date), Folder: v.Category.Term}
isItem = true
} else if t.Name.Local == "title" && ((root == "rss" && depth == 3) || (root == "feed" && depth == 2)) {
if d.DecodeElement(&title, &t) != nil {
return nil, errFeedInvalid
}
depth--
}
if isItem {
if len(out) >= 500 {
return nil, errors.New("limit")
}
item.Title = feedText(item.Title, 2048)
if htmlBody {
item.BodyHTML = feedHTML(item.Body, base)
item.Body = feedText(item.Body, 32768)
} else {
// Atom type=text is literal, even when it contains HTML-looking text.
item.Body = feedText(html.EscapeString(item.Body), 32768)
}
item.Feed = feedText(title, 512)
item.Folder = feedText(item.Folder, 512)
if item.ID == "" {
key := item.URL
if key == "" {
key = item.Title + "\x00" + strconv.FormatInt(item.PublishedMS, 10) + "\x00" + item.Body
}
h := sha256.Sum256([]byte(key))
item.ID = "sha256:" + hex.EncodeToString(h[:])
}
if len(item.ID) > 1024 || seen[item.ID] {
return nil, errFeedInvalid
}
seen[item.ID] = true
out = append(out, item)
}
case xml.EndElement:
depth--
}
}
if root == "" || depth != 0 {
return nil, errFeedInvalid
}
return out, nil
}
func feedRequest(ctx context.Context, client *http.Client, method, target, auth string, form url.Values, headers map[string]string) ([]byte, http.Header, int, error) {
var body io.Reader
if form != nil {
body = strings.NewReader(form.Encode())
}
r, e := http.NewRequestWithContext(ctx, method, target, body)
if e != nil {
return nil, nil, 0, errFeedInvalid
}
if auth != "" {
r.Header.Set("Authorization", "GoogleLogin auth="+auth)
}
if form != nil {
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
r.Header.Set("User-Agent", "PagesWiki-Feeds/0.1")
for k, v := range headers {
r.Header.Set(k, v)
}
res, e := client.Do(r)
if e != nil {
return nil, nil, 0, errors.New("network")
}
defer res.Body.Close()
if res.StatusCode == 401 || res.StatusCode == 403 {
return nil, nil, res.StatusCode, errors.New("credentials")
}
if res.StatusCode == 304 {
return nil, res.Header, 304, nil
}
if res.StatusCode != 200 {
return nil, nil, res.StatusCode, errors.New("remote_error")
}
raw, e := io.ReadAll(io.LimitReader(res.Body, (4<<20)+1))
if e != nil {
return nil, nil, 0, errors.New("network")
}
if len(raw) > 4<<20 {
return nil, nil, 0, errors.New("limit")
}
return raw, res.Header, res.StatusCode, nil
}
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) {
base := strings.TrimRight(v.URL, "/")
raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil)
if e != nil {
return nil, e
}
auth := ""
for _, line := range strings.Split(string(raw), "\n") {
if strings.HasPrefix(line, "Auth=") {
auth = strings.TrimSpace(strings.TrimPrefix(line, "Auth="))
}
}
if auth == "" || len(auth) > 4096 || strings.ContainsAny(auth, "\r\n") {
return nil, errors.New("credentials")
}
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/stream/contents/reading-list?output=json&n=200", auth, nil, nil)
if e != nil {
return nil, e
}
var response struct {
Items []struct {
ID string `json:"id"`
Title string `json:"title"`
Published int64 `json:"published"`
Categories []string `json:"categories"`
Origin struct {
Title string `json:"title"`
StreamID string `json:"streamId"`
} `json:"origin"`
Alternate []struct {
Href string `json:"href"`
} `json:"alternate"`
Summary struct {
Content string `json:"content"`
} `json:"summary"`
Content struct {
Content string `json:"content"`
} `json:"content"`
} `json:"items"`
}
if json.Unmarshal(raw, &response) != nil || response.Items == nil || len(response.Items) > 200 {
return nil, errFeedInvalid
}
out := []store.FeedItem{}
seen := map[string]bool{}
for _, i := range response.Items {
if i.ID == "" || len(i.ID) > 1024 || seen[i.ID] {
return nil, errFeedInvalid
}
seen[i.ID] = true
if (i.Origin.StreamID != "" && !store.ValidFeedRemoteID(i.Origin.StreamID)) || len(i.Categories) > 64 {
return nil, errFeedInvalid
}
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512), RemoteFeed: i.Origin.StreamID, RemoteCategories: []string{}}
if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() {
item.PublishedMS = i.Published * 1000
}
body := i.Summary.Content
if body == "" {
body = i.Content.Content
}
item.Body = feedText(body, 32768)
item.BodyHTML = feedHTML(body, base)
if len(i.Alternate) > 0 {
item.URL = feedLink(base, i.Alternate[0].Href)
}
for _, c := range i.Categories {
if !store.ValidFeedRemoteID(c) {
return nil, errFeedInvalid
}
if strings.HasSuffix(c, "/state/com.google/read") {
item.Read = true
}
if strings.HasSuffix(c, "/state/com.google/starred") {
item.Starred = true
}
if pos := strings.Index(c, "/label/"); pos >= 0 {
item.Folder = feedText(c[pos+7:], 512)
item.RemoteCategories = append(item.RemoteCategories, c)
}
}
out = append(out, item)
}
if len(catalogs) > 0 {
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/subscription/list?output=json", auth, nil, nil)
if e != nil {
return nil, e
}
catalog, e := parseFreshRSSCatalog(raw)
if e != nil {
return nil, e
}
*catalogs[0] = catalog
}
return out, nil
}
func parseFreshRSSCatalog(raw []byte) ([]store.FeedRemoteSubscription, error) {
if len(raw) > 4<<20 || !utf8.Valid(raw) {
return nil, errFeedInvalid
}
var response struct {
Subscriptions []store.FeedRemoteSubscription `json:"subscriptions"`
}
if json.Unmarshal(raw, &response) != nil || response.Subscriptions == nil {
return nil, errFeedInvalid
}
if len(response.Subscriptions) > 1000 {
return nil, errors.New("limit")
}
for i := range response.Subscriptions {
sub := &response.Subscriptions[i]
sub.Title = feedText(sub.Title, 512)
if sub.Categories == nil {
sub.Categories = []store.FeedCategory{}
}
for j := range sub.Categories {
sub.Categories[j].Label = feedText(sub.Categories[j].Label, 512)
}
}
if _, e := store.FeedCatalogJSON(response.Subscriptions); e != nil {
return nil, errFeedInvalid
}
return response.Subscriptions, nil
}
func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error {
client := calendarSourceClient()
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
defer client.CloseIdleConnections()
return s.syncFeedSourcesWithClient(ctx, job, client)
}
// Tests inject a synthetic transport; production always uses the DNS-pinned,
// public HTTPS-only transport above, never a user-configurable HTTP client.
func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.EventJob, client *http.Client) error {
ctx, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
sources, e := s.store.DueFeedSources(ctx)
if e != nil {
return e
}
done, failed := 0, 0
for _, v := range sources {
if ctx.Err() != nil {
break
}
if !s.systemExtensionEnabled(v.Domain, "organizer.feeds") {
continue
}
v, e = s.store.ClaimFeed(ctx, v)
if e != nil {
continue
}
child, stop := context.WithTimeout(ctx, 8*time.Second)
var items []store.FeedItem
var catalog []store.FeedRemoteSubscription
etag, modified := v.ETag, v.Modified
unchanged := false
if !validCalendarSourceURL(v.URL) {
e = errFeedInvalid
} else if v.Provider == "freshrss" {
var user, password string
user, password, e = s.store.OpenFeedCredential(v)
if e == nil {
items, e = fetchFreshRSS(child, client, v, user, password, &catalog)
} else {
e = errors.New("credentials")
}
} else {
var raw []byte
var h http.Header
var status int
raw, h, status, e = feedRequest(child, client, "GET", v.URL, "", nil, map[string]string{"If-None-Match": v.ETag, "If-Modified-Since": v.Modified})
if e == nil {
unchanged = status == 304
if !unchanged {
items, e = parseFeed(raw, v.URL)
etag, modified = h.Get("ETag"), h.Get("Last-Modified")
}
}
}
stop()
if e == nil && (len(etag) > 1024 || len(modified) > 1024) {
e = errors.New("limit")
etag, modified = v.ETag, v.Modified
items = nil
}
code := ""
if e != nil {
code = e.Error()
switch code {
case "network", "invalid_feed", "credentials", "remote_error", "limit":
default:
code = "invalid_feed"
}
failed++
} else {
done++
}
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged, catalog); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
return e
}
}
s.addEventProgressLog(ctx, job, fmt.Sprintf("feed sources updated=%d failed=%d", done, failed))
return nil
}