package server import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "html" "io" "net/http" "net/url" "strconv" "strings" "time" "unicode/utf8" "github.com/microcosm-cc/bluemonday" xhtml "golang.org/x/net/html" "golang.org/x/net/html/charset" "pageswiki/internal/store" ) var errFeedInvalid = errors.New("invalid_feed") var feedTextPolicy = bluemonday.StrictPolicy() var feedHTMLPolicy = func() *bluemonday.Policy { p := bluemonday.NewPolicy() p.AllowElements("p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "b", "em", "i", "u", "s", "del", "ins", "sub", "sup", "mark", "small", "blockquote", "pre", "code", "kbd", "samp", "ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td", "a") p.AllowAttrs("href", "target", "rel").OnElements("a") p.AllowAttrs("colspan", "rowspan", "scope").OnElements("th", "td") p.AllowStandardURLs() p.AllowRelativeURLs(false) return p }() // Keep article structure without granting fetched HTML script, style, form, // embedding or automatic network capabilities. Wiki capture still uses Body. func feedHTML(raw, base string) string { if raw == "" || len(raw) > 128<<10 { return "" } z := xhtml.NewTokenizer(strings.NewReader(raw)) var out strings.Builder markup := false for { typeOf := z.Next() if typeOf == xhtml.ErrorToken { if z.Err() != io.EOF { return "" } break } if typeOf == xhtml.StartTagToken || typeOf == xhtml.SelfClosingTagToken { markup = true tok := z.Token() if tok.Data == "a" { attrs := []xhtml.Attribute{} for _, attr := range tok.Attr { if attr.Key == "href" { if link := feedLink(base, attr.Val); link != "" { attrs = append(attrs, xhtml.Attribute{Key: "href", Val: link}) } } } tok.Attr = append(attrs, xhtml.Attribute{Key: "target", Val: "_blank"}, xhtml.Attribute{Key: "rel", Val: "noopener noreferrer"}) } out.WriteString(tok.String()) } else { out.Write(z.Raw()) } } if !markup { return "" } clean := feedHTMLPolicy.Sanitize(out.String()) if len(clean) > 128<<10 { return "" } return clean } func feedText(raw string, limit int) string { // Plain text for fallback and inert Wiki/task capture. for _, tag := range []string{"

", "", "
", "
", "
", ""} { raw = strings.ReplaceAll(raw, tag, tag+"\n") } text := strings.TrimSpace(html.UnescapeString(feedTextPolicy.Sanitize(raw))) if len(text) > limit { text = text[:limit-16] for !utf8.ValidString(text) { text = text[:len(text)-1] } text += "\n[… truncated]" } return text } func feedLink(base, raw string) string { if strings.TrimSpace(raw) == "" || len(raw) > 4096 { return "" } u, e := url.Parse(strings.TrimSpace(raw)) if e != nil { return "" } b, e := url.Parse(base) if e != nil { return "" } u = b.ResolveReference(u) if (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil { return "" } result := u.String() if len(result) > 4096 { return "" } return result } func feedDate(v string) int64 { for _, layout := range []string{time.RFC3339Nano, time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, "2006-01-02T15:04:05-0700"} { if t, e := time.Parse(layout, v); e == nil && t.UnixMilli() > 0 && t.Before(time.Now().AddDate(1, 0, 0)) { return t.UnixMilli() } } return 0 } type rssEntry struct { ID string `xml:"guid"` Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` Content string `xml:"encoded"` Date string `xml:"pubDate"` Category string `xml:"category"` } type atomText struct { Type string `xml:"type,attr"` Text string `xml:",chardata"` Inner string `xml:",innerxml"` } func (v atomText) body() (string, bool) { if v.Type == "xhtml" { return v.Inner, true } return v.Text, v.Type == "html" } type atomEntry struct { ID string `xml:"id"` Title string `xml:"title"` Links []struct { Href string `xml:"href,attr"` Rel string `xml:"rel,attr"` } `xml:"link"` Summary atomText `xml:"summary"` Content atomText `xml:"content"` Published string `xml:"published"` Updated string `xml:"updated"` Category struct { Term string `xml:"term,attr"` } `xml:"category"` } func parseFeed(raw []byte, base string) ([]store.FeedItem, error) { if len(raw) > 4<<20 { return nil, errors.New("limit") } d := xml.NewDecoder(bytes.NewReader(raw)) d.CharsetReader = charset.NewReaderLabel root := "" title := "" depth := 0 out := []store.FeedItem{} seen := map[string]bool{} for { tok, e := d.Token() if e == io.EOF { break } if e != nil { return nil, errFeedInvalid } switch t := tok.(type) { case xml.Directive: return nil, errFeedInvalid // DTD/entities have no role in a feed. case xml.StartElement: depth++ if depth == 1 && root != "" { return nil, errFeedInvalid } if depth > 64 { return nil, errFeedInvalid } if root == "" { root = t.Name.Local if root != "rss" && root != "feed" { return nil, errFeedInvalid } } var item store.FeedItem isItem := false htmlBody := false if root == "rss" && t.Name.Local == "item" && depth == 3 { var v rssEntry if d.DecodeElement(&v, &t) != nil { return nil, errFeedInvalid } depth-- body := v.Content if body == "" { body = v.Description } item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category} isItem = true htmlBody = true } else if root == "feed" && t.Name.Local == "entry" && depth == 2 { var v atomEntry if d.DecodeElement(&v, &t) != nil { return nil, errFeedInvalid } depth-- body, isHTML := v.Content.body() if body == "" { body, isHTML = v.Summary.body() } htmlBody = isHTML link := "" for _, l := range v.Links { if l.Rel == "" || l.Rel == "alternate" { link = feedLink(base, l.Href) break } } date := v.Published if date == "" { date = v.Updated } item = store.FeedItem{ID: v.ID, Title: v.Title, URL: link, Body: body, PublishedMS: feedDate(date), Folder: v.Category.Term} isItem = true } else if t.Name.Local == "title" && ((root == "rss" && depth == 3) || (root == "feed" && depth == 2)) { if d.DecodeElement(&title, &t) != nil { return nil, errFeedInvalid } depth-- } if isItem { if len(out) >= 500 { return nil, errors.New("limit") } item.Title = feedText(item.Title, 2048) if htmlBody { item.BodyHTML = feedHTML(item.Body, base) item.Body = feedText(item.Body, 32768) } else { // Atom type=text is literal, even when it contains HTML-looking text. item.Body = feedText(html.EscapeString(item.Body), 32768) } item.Feed = feedText(title, 512) item.Folder = feedText(item.Folder, 512) if item.ID == "" { key := item.URL if key == "" { key = item.Title + "\x00" + strconv.FormatInt(item.PublishedMS, 10) + "\x00" + item.Body } h := sha256.Sum256([]byte(key)) item.ID = "sha256:" + hex.EncodeToString(h[:]) } if len(item.ID) > 1024 || seen[item.ID] { return nil, errFeedInvalid } seen[item.ID] = true out = append(out, item) } case xml.EndElement: depth-- } } if root == "" || depth != 0 { return nil, errFeedInvalid } return out, nil } func feedRequest(ctx context.Context, client *http.Client, method, target, auth string, form url.Values, headers map[string]string) ([]byte, http.Header, int, error) { var body io.Reader if form != nil { body = strings.NewReader(form.Encode()) } r, e := http.NewRequestWithContext(ctx, method, target, body) if e != nil { return nil, nil, 0, errFeedInvalid } if auth != "" { r.Header.Set("Authorization", "GoogleLogin auth="+auth) } if form != nil { r.Header.Set("Content-Type", "application/x-www-form-urlencoded") } r.Header.Set("User-Agent", "PagesWiki-Feeds/0.1") for k, v := range headers { r.Header.Set(k, v) } res, e := client.Do(r) if e != nil { return nil, nil, 0, errors.New("network") } defer res.Body.Close() if res.StatusCode == 401 || res.StatusCode == 403 { return nil, nil, res.StatusCode, errors.New("credentials") } if res.StatusCode == 304 { return nil, res.Header, 304, nil } if res.StatusCode != 200 { return nil, nil, res.StatusCode, errors.New("remote_error") } raw, e := io.ReadAll(io.LimitReader(res.Body, (4<<20)+1)) if e != nil { return nil, nil, 0, errors.New("network") } if len(raw) > 4<<20 { return nil, nil, 0, errors.New("limit") } return raw, res.Header, res.StatusCode, nil } func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) { base := strings.TrimRight(v.URL, "/") raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil) if e != nil { return nil, e } auth := "" for _, line := range strings.Split(string(raw), "\n") { if strings.HasPrefix(line, "Auth=") { auth = strings.TrimSpace(strings.TrimPrefix(line, "Auth=")) } } if auth == "" || len(auth) > 4096 || strings.ContainsAny(auth, "\r\n") { return nil, errors.New("credentials") } raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/stream/contents/reading-list?output=json&n=200", auth, nil, nil) if e != nil { return nil, e } var response struct { Items []struct { ID string `json:"id"` Title string `json:"title"` Published int64 `json:"published"` Categories []string `json:"categories"` Origin struct { Title string `json:"title"` StreamID string `json:"streamId"` } `json:"origin"` Alternate []struct { Href string `json:"href"` } `json:"alternate"` Summary struct { Content string `json:"content"` } `json:"summary"` Content struct { Content string `json:"content"` } `json:"content"` } `json:"items"` } if json.Unmarshal(raw, &response) != nil || response.Items == nil || len(response.Items) > 200 { return nil, errFeedInvalid } out := []store.FeedItem{} seen := map[string]bool{} for _, i := range response.Items { if i.ID == "" || len(i.ID) > 1024 || seen[i.ID] { return nil, errFeedInvalid } seen[i.ID] = true if (i.Origin.StreamID != "" && !store.ValidFeedRemoteID(i.Origin.StreamID)) || len(i.Categories) > 64 { return nil, errFeedInvalid } item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512), RemoteFeed: i.Origin.StreamID, RemoteCategories: []string{}} if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() { item.PublishedMS = i.Published * 1000 } body := i.Summary.Content if body == "" { body = i.Content.Content } item.Body = feedText(body, 32768) item.BodyHTML = feedHTML(body, base) if len(i.Alternate) > 0 { item.URL = feedLink(base, i.Alternate[0].Href) } for _, c := range i.Categories { if !store.ValidFeedRemoteID(c) { return nil, errFeedInvalid } if strings.HasSuffix(c, "/state/com.google/read") { item.Read = true } if strings.HasSuffix(c, "/state/com.google/starred") { item.Starred = true } if pos := strings.Index(c, "/label/"); pos >= 0 { item.Folder = feedText(c[pos+7:], 512) item.RemoteCategories = append(item.RemoteCategories, c) } } out = append(out, item) } if len(catalogs) > 0 { raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/subscription/list?output=json", auth, nil, nil) if e != nil { return nil, e } catalog, e := parseFreshRSSCatalog(raw) if e != nil { return nil, e } *catalogs[0] = catalog } return out, nil } func parseFreshRSSCatalog(raw []byte) ([]store.FeedRemoteSubscription, error) { if len(raw) > 4<<20 || !utf8.Valid(raw) { return nil, errFeedInvalid } var response struct { Subscriptions []store.FeedRemoteSubscription `json:"subscriptions"` } if json.Unmarshal(raw, &response) != nil || response.Subscriptions == nil { return nil, errFeedInvalid } if len(response.Subscriptions) > 1000 { return nil, errors.New("limit") } for i := range response.Subscriptions { sub := &response.Subscriptions[i] sub.Title = feedText(sub.Title, 512) if sub.Categories == nil { sub.Categories = []store.FeedCategory{} } for j := range sub.Categories { sub.Categories[j].Label = feedText(sub.Categories[j].Label, 512) } } if _, e := store.FeedCatalogJSON(response.Subscriptions); e != nil { return nil, errFeedInvalid } return response.Subscriptions, nil } func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error { client := calendarSourceClient() client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } defer client.CloseIdleConnections() return s.syncFeedSourcesWithClient(ctx, job, client) } // Tests inject a synthetic transport; production always uses the DNS-pinned, // public HTTPS-only transport above, never a user-configurable HTTP client. func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.EventJob, client *http.Client) error { ctx, cancel := context.WithTimeout(ctx, 35*time.Second) defer cancel() sources, e := s.store.DueFeedSources(ctx) if e != nil { return e } done, failed := 0, 0 for _, v := range sources { if ctx.Err() != nil { break } if !s.systemExtensionEnabled(v.Domain, "organizer.feeds") { continue } v, e = s.store.ClaimFeed(ctx, v) if e != nil { continue } child, stop := context.WithTimeout(ctx, 8*time.Second) var items []store.FeedItem var catalog []store.FeedRemoteSubscription etag, modified := v.ETag, v.Modified unchanged := false if !validCalendarSourceURL(v.URL) { e = errFeedInvalid } else if v.Provider == "freshrss" { var user, password string user, password, e = s.store.OpenFeedCredential(v) if e == nil { items, e = fetchFreshRSS(child, client, v, user, password, &catalog) } else { e = errors.New("credentials") } } else { var raw []byte var h http.Header var status int raw, h, status, e = feedRequest(child, client, "GET", v.URL, "", nil, map[string]string{"If-None-Match": v.ETag, "If-Modified-Since": v.Modified}) if e == nil { unchanged = status == 304 if !unchanged { items, e = parseFeed(raw, v.URL) etag, modified = h.Get("ETag"), h.Get("Last-Modified") } } } stop() if e == nil && (len(etag) > 1024 || len(modified) > 1024) { e = errors.New("limit") etag, modified = v.ETag, v.Modified items = nil } code := "" if e != nil { code = e.Error() switch code { case "network", "invalid_feed", "credentials", "remote_error", "limit": default: code = "invalid_feed" } failed++ } else { done++ } if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged, catalog); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) { return e } } s.addEventProgressLog(ctx, job, fmt.Sprintf("feed sources updated=%d failed=%d", done, failed)) return nil }