574 lines
16 KiB
Go
574 lines
16 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"github.com/microcosm-cc/bluemonday"
|
|
xhtml "golang.org/x/net/html"
|
|
"golang.org/x/net/html/charset"
|
|
"pageswiki/internal/store"
|
|
)
|
|
|
|
var errFeedInvalid = errors.New("invalid_feed")
|
|
var feedTextPolicy = bluemonday.StrictPolicy()
|
|
var feedHTMLPolicy = func() *bluemonday.Policy {
|
|
p := bluemonday.NewPolicy()
|
|
p.AllowElements("p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "b", "em", "i", "u", "s", "del", "ins", "sub", "sup", "mark", "small", "blockquote", "pre", "code", "kbd", "samp", "ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td", "a")
|
|
p.AllowAttrs("href", "target", "rel").OnElements("a")
|
|
p.AllowAttrs("colspan", "rowspan", "scope").OnElements("th", "td")
|
|
p.AllowStandardURLs()
|
|
p.AllowRelativeURLs(false)
|
|
return p
|
|
}()
|
|
|
|
// Keep article structure without granting fetched HTML script, style, form,
|
|
// embedding or automatic network capabilities. Wiki capture still uses Body.
|
|
func feedHTML(raw, base string) string {
|
|
if raw == "" || len(raw) > 128<<10 {
|
|
return ""
|
|
}
|
|
z := xhtml.NewTokenizer(strings.NewReader(raw))
|
|
var out strings.Builder
|
|
markup := false
|
|
for {
|
|
typeOf := z.Next()
|
|
if typeOf == xhtml.ErrorToken {
|
|
if z.Err() != io.EOF {
|
|
return ""
|
|
}
|
|
break
|
|
}
|
|
if typeOf == xhtml.StartTagToken || typeOf == xhtml.SelfClosingTagToken {
|
|
markup = true
|
|
tok := z.Token()
|
|
if tok.Data == "a" {
|
|
attrs := []xhtml.Attribute{}
|
|
for _, attr := range tok.Attr {
|
|
if attr.Key == "href" {
|
|
if link := feedLink(base, attr.Val); link != "" {
|
|
attrs = append(attrs, xhtml.Attribute{Key: "href", Val: link})
|
|
}
|
|
}
|
|
}
|
|
tok.Attr = append(attrs, xhtml.Attribute{Key: "target", Val: "_blank"}, xhtml.Attribute{Key: "rel", Val: "noopener noreferrer"})
|
|
}
|
|
out.WriteString(tok.String())
|
|
} else {
|
|
out.Write(z.Raw())
|
|
}
|
|
}
|
|
if !markup {
|
|
return ""
|
|
}
|
|
clean := feedHTMLPolicy.Sanitize(out.String())
|
|
if len(clean) > 128<<10 {
|
|
return ""
|
|
}
|
|
return clean
|
|
}
|
|
|
|
func feedText(raw string, limit int) string {
|
|
// Plain text for fallback and inert Wiki/task capture.
|
|
for _, tag := range []string{"</p>", "</div>", "<br>", "<br/>", "<br />", "</li>"} {
|
|
raw = strings.ReplaceAll(raw, tag, tag+"\n")
|
|
}
|
|
text := strings.TrimSpace(html.UnescapeString(feedTextPolicy.Sanitize(raw)))
|
|
if len(text) > limit {
|
|
text = text[:limit-16]
|
|
for !utf8.ValidString(text) {
|
|
text = text[:len(text)-1]
|
|
}
|
|
text += "\n[… truncated]"
|
|
}
|
|
return text
|
|
}
|
|
func feedLink(base, raw string) string {
|
|
if strings.TrimSpace(raw) == "" || len(raw) > 4096 {
|
|
return ""
|
|
}
|
|
u, e := url.Parse(strings.TrimSpace(raw))
|
|
if e != nil {
|
|
return ""
|
|
}
|
|
b, e := url.Parse(base)
|
|
if e != nil {
|
|
return ""
|
|
}
|
|
u = b.ResolveReference(u)
|
|
if (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
|
|
return ""
|
|
}
|
|
result := u.String()
|
|
if len(result) > 4096 {
|
|
return ""
|
|
}
|
|
return result
|
|
}
|
|
func feedDate(v string) int64 {
|
|
for _, layout := range []string{time.RFC3339Nano, time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, "2006-01-02T15:04:05-0700"} {
|
|
if t, e := time.Parse(layout, v); e == nil && t.UnixMilli() > 0 && t.Before(time.Now().AddDate(1, 0, 0)) {
|
|
return t.UnixMilli()
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
type rssEntry struct {
|
|
ID string `xml:"guid"`
|
|
Title string `xml:"title"`
|
|
Link string `xml:"link"`
|
|
Description string `xml:"description"`
|
|
Content string `xml:"encoded"`
|
|
Date string `xml:"pubDate"`
|
|
Category string `xml:"category"`
|
|
}
|
|
type atomText struct {
|
|
Type string `xml:"type,attr"`
|
|
Text string `xml:",chardata"`
|
|
Inner string `xml:",innerxml"`
|
|
}
|
|
|
|
func (v atomText) body() (string, bool) {
|
|
if v.Type == "xhtml" {
|
|
return v.Inner, true
|
|
}
|
|
return v.Text, v.Type == "html"
|
|
}
|
|
|
|
type atomEntry struct {
|
|
ID string `xml:"id"`
|
|
Title string `xml:"title"`
|
|
Links []struct {
|
|
Href string `xml:"href,attr"`
|
|
Rel string `xml:"rel,attr"`
|
|
} `xml:"link"`
|
|
Summary atomText `xml:"summary"`
|
|
Content atomText `xml:"content"`
|
|
Published string `xml:"published"`
|
|
Updated string `xml:"updated"`
|
|
Category struct {
|
|
Term string `xml:"term,attr"`
|
|
} `xml:"category"`
|
|
}
|
|
|
|
func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
|
if len(raw) > 4<<20 {
|
|
return nil, errors.New("limit")
|
|
}
|
|
d := xml.NewDecoder(bytes.NewReader(raw))
|
|
d.CharsetReader = charset.NewReaderLabel
|
|
root := ""
|
|
title := ""
|
|
depth := 0
|
|
out := []store.FeedItem{}
|
|
seen := map[string]bool{}
|
|
for {
|
|
tok, e := d.Token()
|
|
if e == io.EOF {
|
|
break
|
|
}
|
|
if e != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
switch t := tok.(type) {
|
|
case xml.Directive:
|
|
return nil, errFeedInvalid // DTD/entities have no role in a feed.
|
|
case xml.StartElement:
|
|
depth++
|
|
if depth == 1 && root != "" {
|
|
return nil, errFeedInvalid
|
|
}
|
|
if depth > 64 {
|
|
return nil, errFeedInvalid
|
|
}
|
|
if root == "" {
|
|
root = t.Name.Local
|
|
if root != "rss" && root != "feed" {
|
|
return nil, errFeedInvalid
|
|
}
|
|
}
|
|
var item store.FeedItem
|
|
isItem := false
|
|
htmlBody := false
|
|
if root == "rss" && t.Name.Local == "item" && depth == 3 {
|
|
var v rssEntry
|
|
if d.DecodeElement(&v, &t) != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
depth--
|
|
body := v.Content
|
|
if body == "" {
|
|
body = v.Description
|
|
}
|
|
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category}
|
|
isItem = true
|
|
htmlBody = true
|
|
} else if root == "feed" && t.Name.Local == "entry" && depth == 2 {
|
|
var v atomEntry
|
|
if d.DecodeElement(&v, &t) != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
depth--
|
|
body, isHTML := v.Content.body()
|
|
if body == "" {
|
|
body, isHTML = v.Summary.body()
|
|
}
|
|
htmlBody = isHTML
|
|
link := ""
|
|
for _, l := range v.Links {
|
|
if l.Rel == "" || l.Rel == "alternate" {
|
|
link = feedLink(base, l.Href)
|
|
break
|
|
}
|
|
}
|
|
date := v.Published
|
|
if date == "" {
|
|
date = v.Updated
|
|
}
|
|
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: link, Body: body, PublishedMS: feedDate(date), Folder: v.Category.Term}
|
|
isItem = true
|
|
} else if t.Name.Local == "title" && ((root == "rss" && depth == 3) || (root == "feed" && depth == 2)) {
|
|
if d.DecodeElement(&title, &t) != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
depth--
|
|
}
|
|
if isItem {
|
|
if len(out) >= 500 {
|
|
return nil, errors.New("limit")
|
|
}
|
|
item.Title = feedText(item.Title, 2048)
|
|
if htmlBody {
|
|
item.BodyHTML = feedHTML(item.Body, base)
|
|
item.Body = feedText(item.Body, 32768)
|
|
} else {
|
|
// Atom type=text is literal, even when it contains HTML-looking text.
|
|
item.Body = feedText(html.EscapeString(item.Body), 32768)
|
|
}
|
|
item.Feed = feedText(title, 512)
|
|
item.Folder = feedText(item.Folder, 512)
|
|
if item.ID == "" {
|
|
key := item.URL
|
|
if key == "" {
|
|
key = item.Title + "\x00" + strconv.FormatInt(item.PublishedMS, 10) + "\x00" + item.Body
|
|
}
|
|
h := sha256.Sum256([]byte(key))
|
|
item.ID = "sha256:" + hex.EncodeToString(h[:])
|
|
}
|
|
if len(item.ID) > 1024 || seen[item.ID] {
|
|
return nil, errFeedInvalid
|
|
}
|
|
seen[item.ID] = true
|
|
out = append(out, item)
|
|
}
|
|
case xml.EndElement:
|
|
depth--
|
|
}
|
|
}
|
|
if root == "" || depth != 0 {
|
|
return nil, errFeedInvalid
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func feedRequest(ctx context.Context, client *http.Client, method, target, auth string, form url.Values, headers map[string]string) ([]byte, http.Header, int, error) {
|
|
var body io.Reader
|
|
if form != nil {
|
|
body = strings.NewReader(form.Encode())
|
|
}
|
|
r, e := http.NewRequestWithContext(ctx, method, target, body)
|
|
if e != nil {
|
|
return nil, nil, 0, errFeedInvalid
|
|
}
|
|
if auth != "" {
|
|
r.Header.Set("Authorization", "GoogleLogin auth="+auth)
|
|
}
|
|
if form != nil {
|
|
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
}
|
|
r.Header.Set("User-Agent", "PagesWiki-Feeds/0.1")
|
|
for k, v := range headers {
|
|
r.Header.Set(k, v)
|
|
}
|
|
res, e := client.Do(r)
|
|
if e != nil {
|
|
return nil, nil, 0, errors.New("network")
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode == 401 || res.StatusCode == 403 {
|
|
return nil, nil, res.StatusCode, errors.New("credentials")
|
|
}
|
|
if res.StatusCode == 304 {
|
|
return nil, res.Header, 304, nil
|
|
}
|
|
if res.StatusCode != 200 {
|
|
return nil, nil, res.StatusCode, errors.New("remote_error")
|
|
}
|
|
raw, e := io.ReadAll(io.LimitReader(res.Body, (4<<20)+1))
|
|
if e != nil {
|
|
return nil, nil, 0, errors.New("network")
|
|
}
|
|
if len(raw) > 4<<20 {
|
|
return nil, nil, 0, errors.New("limit")
|
|
}
|
|
return raw, res.Header, res.StatusCode, nil
|
|
}
|
|
func freshRSSLogin(ctx context.Context, client *http.Client, v store.FeedSource, user, password string) (string, error) {
|
|
base := strings.TrimRight(v.URL, "/")
|
|
raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
auth := ""
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
if strings.HasPrefix(line, "Auth=") {
|
|
auth = strings.TrimSpace(strings.TrimPrefix(line, "Auth="))
|
|
}
|
|
}
|
|
if auth == "" || len(auth) > 4096 || strings.ContainsAny(auth, "\r\n") {
|
|
return "", errors.New("credentials")
|
|
}
|
|
return auth, nil
|
|
}
|
|
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*store.FeedCatalogSnapshot) ([]store.FeedItem, error) {
|
|
base := strings.TrimRight(v.URL, "/")
|
|
auth, e := freshRSSLogin(ctx, client, v, user, password)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
raw, _, _, e := feedRequest(ctx, client, "GET", base+"/reader/api/0/stream/contents/reading-list?output=json&n=200", auth, nil, nil)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
items, e := parseFreshRSSItems(raw, base)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if len(catalogs) > 0 {
|
|
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/subscription/list?output=json", auth, nil, nil)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
catalog, e := parseFreshRSSCatalog(raw)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/tag/list?output=json", auth, nil, nil)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
folders, e := parseFreshRSSFolders(raw)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
folders, e = store.FeedFolders(catalog, folders)
|
|
if e != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
snapshot := store.FeedCatalogSnapshot{Subscriptions: catalog, Folders: folders}
|
|
if _, _, e = snapshot.Encode(); e != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
*catalogs[0] = snapshot
|
|
}
|
|
return items, nil
|
|
}
|
|
func parseFreshRSSItems(raw []byte, base string) ([]store.FeedItem, error) {
|
|
var response struct {
|
|
Items []struct {
|
|
ID string `json:"id"`
|
|
Title string `json:"title"`
|
|
Published int64 `json:"published"`
|
|
Categories []string `json:"categories"`
|
|
Origin struct {
|
|
Title string `json:"title"`
|
|
StreamID string `json:"streamId"`
|
|
} `json:"origin"`
|
|
Alternate []struct {
|
|
Href string `json:"href"`
|
|
} `json:"alternate"`
|
|
Summary struct {
|
|
Content string `json:"content"`
|
|
} `json:"summary"`
|
|
Content struct {
|
|
Content string `json:"content"`
|
|
} `json:"content"`
|
|
} `json:"items"`
|
|
}
|
|
if json.Unmarshal(raw, &response) != nil || response.Items == nil || len(response.Items) > 200 {
|
|
return nil, errFeedInvalid
|
|
}
|
|
out := []store.FeedItem{}
|
|
seen := map[string]bool{}
|
|
for _, i := range response.Items {
|
|
if i.ID == "" || len(i.ID) > 1024 || seen[i.ID] {
|
|
return nil, errFeedInvalid
|
|
}
|
|
seen[i.ID] = true
|
|
if (i.Origin.StreamID != "" && !store.ValidFeedRemoteID(i.Origin.StreamID)) || len(i.Categories) > 64 {
|
|
return nil, errFeedInvalid
|
|
}
|
|
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512), RemoteFeed: i.Origin.StreamID, RemoteCategories: []string{}}
|
|
if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() {
|
|
item.PublishedMS = i.Published * 1000
|
|
}
|
|
body := i.Summary.Content
|
|
if body == "" {
|
|
body = i.Content.Content
|
|
}
|
|
item.Body = feedText(body, 32768)
|
|
item.BodyHTML = feedHTML(body, base)
|
|
if len(i.Alternate) > 0 {
|
|
item.URL = feedLink(base, i.Alternate[0].Href)
|
|
}
|
|
for _, c := range i.Categories {
|
|
if !store.ValidFeedRemoteID(c) {
|
|
return nil, errFeedInvalid
|
|
}
|
|
if strings.HasSuffix(c, "/state/com.google/read") {
|
|
item.Read = true
|
|
}
|
|
if strings.HasSuffix(c, "/state/com.google/starred") {
|
|
item.Starred = true
|
|
}
|
|
if pos := strings.Index(c, "/label/"); pos >= 0 {
|
|
item.Folder = feedText(c[pos+7:], 512)
|
|
item.RemoteCategories = append(item.RemoteCategories, c)
|
|
}
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func parseFreshRSSCatalog(raw []byte) ([]store.FeedRemoteSubscription, error) {
|
|
if len(raw) > 4<<20 || !utf8.Valid(raw) {
|
|
return nil, errFeedInvalid
|
|
}
|
|
var response struct {
|
|
Subscriptions []store.FeedRemoteSubscription `json:"subscriptions"`
|
|
}
|
|
if json.Unmarshal(raw, &response) != nil || response.Subscriptions == nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
if len(response.Subscriptions) > 1000 {
|
|
return nil, errors.New("limit")
|
|
}
|
|
for i := range response.Subscriptions {
|
|
sub := &response.Subscriptions[i]
|
|
sub.Title = feedText(sub.Title, 512)
|
|
if sub.Categories == nil {
|
|
sub.Categories = []store.FeedCategory{}
|
|
}
|
|
for j := range sub.Categories {
|
|
sub.Categories[j].Label = feedText(sub.Categories[j].Label, 512)
|
|
}
|
|
}
|
|
if _, e := store.FeedCatalogJSON(response.Subscriptions); e != nil {
|
|
return nil, errFeedInvalid
|
|
}
|
|
return response.Subscriptions, nil
|
|
}
|
|
func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error {
|
|
client := calendarSourceClient()
|
|
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
|
defer client.CloseIdleConnections()
|
|
return s.syncFeedSourcesWithClient(ctx, job, client)
|
|
}
|
|
|
|
// Tests inject a synthetic transport; production always uses the DNS-pinned,
|
|
// public HTTPS-only transport above, never a user-configurable HTTP client.
|
|
func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.EventJob, client *http.Client) error {
|
|
ctx, cancel := context.WithTimeout(ctx, 35*time.Second)
|
|
defer cancel()
|
|
if e := s.syncFeedActions(ctx, client); e != nil {
|
|
return e
|
|
}
|
|
sources, e := s.store.DueFeedSources(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
done, failed := 0, 0
|
|
for _, v := range sources {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < 9*time.Second {
|
|
break
|
|
}
|
|
if !s.systemExtensionEnabled(v.Domain, "organizer.feeds") {
|
|
continue
|
|
}
|
|
v, e = s.store.ClaimFeed(ctx, v)
|
|
if e != nil {
|
|
continue
|
|
}
|
|
child, stop := context.WithTimeout(ctx, 8*time.Second)
|
|
var items []store.FeedItem
|
|
var catalog store.FeedCatalogSnapshot
|
|
etag, modified := v.ETag, v.Modified
|
|
unchanged := false
|
|
if !validCalendarSourceURL(v.URL) {
|
|
e = errFeedInvalid
|
|
} else if v.Provider == "freshrss" {
|
|
var user, password string
|
|
user, password, e = s.store.OpenFeedCredential(v)
|
|
if e == nil {
|
|
items, e = fetchFreshRSS(child, client, v, user, password, &catalog)
|
|
} else {
|
|
e = errors.New("credentials")
|
|
}
|
|
} else {
|
|
var raw []byte
|
|
var h http.Header
|
|
var status int
|
|
raw, h, status, e = feedRequest(child, client, "GET", v.URL, "", nil, map[string]string{"If-None-Match": v.ETag, "If-Modified-Since": v.Modified})
|
|
if e == nil {
|
|
unchanged = status == 304
|
|
if !unchanged {
|
|
items, e = parseFeed(raw, v.URL)
|
|
etag, modified = h.Get("ETag"), h.Get("Last-Modified")
|
|
}
|
|
}
|
|
}
|
|
stop()
|
|
if e == nil && (len(etag) > 1024 || len(modified) > 1024) {
|
|
e = errors.New("limit")
|
|
etag, modified = v.ETag, v.Modified
|
|
items = nil
|
|
}
|
|
code := ""
|
|
if e != nil {
|
|
code = e.Error()
|
|
switch code {
|
|
case "network", "invalid_feed", "credentials", "remote_error", "limit":
|
|
default:
|
|
code = "invalid_feed"
|
|
}
|
|
failed++
|
|
} else {
|
|
done++
|
|
}
|
|
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged, catalog); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
|
|
return e
|
|
}
|
|
}
|
|
s.addEventProgressLog(ctx, job, fmt.Sprintf("feed sources updated=%d failed=%d", done, failed))
|
|
return nil
|
|
}
|