feat: render RSS HTML bodies and simplify reader navigation
Build and publish container images / images (push) Successful in 12m32s
Build and publish container images / images (push) Successful in 12m32s
This commit is contained in:
1 parent
0d8997972c
commit
38daea737d
12 files changed
+208
-49
No files matched your search
@@ -19,15 +19,71 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
xhtml "golang.org/x/net/html"
|
||||
"golang.org/x/net/html/charset"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
var errFeedInvalid = errors.New("invalid_feed")
|
||||
var feedTextPolicy = bluemonday.StrictPolicy()
|
||||
var feedHTMLPolicy = func() *bluemonday.Policy {
|
||||
p := bluemonday.NewPolicy()
|
||||
p.AllowElements("p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "b", "em", "i", "u", "s", "del", "ins", "sub", "sup", "mark", "small", "blockquote", "pre", "code", "kbd", "samp", "ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td", "a")
|
||||
p.AllowAttrs("href", "target", "rel").OnElements("a")
|
||||
p.AllowAttrs("colspan", "rowspan", "scope").OnElements("th", "td")
|
||||
p.AllowStandardURLs()
|
||||
p.AllowRelativeURLs(false)
|
||||
return p
|
||||
}()
|
||||
|
||||
// Keep article structure without granting fetched HTML script, style, form,
|
||||
// embedding or automatic network capabilities. Wiki capture still uses Body.
|
||||
func feedHTML(raw, base string) string {
|
||||
if raw == "" || len(raw) > 128<<10 {
|
||||
return ""
|
||||
}
|
||||
z := xhtml.NewTokenizer(strings.NewReader(raw))
|
||||
var out strings.Builder
|
||||
markup := false
|
||||
for {
|
||||
typeOf := z.Next()
|
||||
if typeOf == xhtml.ErrorToken {
|
||||
if z.Err() != io.EOF {
|
||||
return ""
|
||||
}
|
||||
break
|
||||
}
|
||||
if typeOf == xhtml.StartTagToken || typeOf == xhtml.SelfClosingTagToken {
|
||||
markup = true
|
||||
tok := z.Token()
|
||||
if tok.Data == "a" {
|
||||
attrs := []xhtml.Attribute{}
|
||||
for _, attr := range tok.Attr {
|
||||
if attr.Key == "href" {
|
||||
if link := feedLink(base, attr.Val); link != "" {
|
||||
attrs = append(attrs, xhtml.Attribute{Key: "href", Val: link})
|
||||
}
|
||||
}
|
||||
}
|
||||
tok.Attr = append(attrs, xhtml.Attribute{Key: "target", Val: "_blank"}, xhtml.Attribute{Key: "rel", Val: "noopener noreferrer"})
|
||||
}
|
||||
out.WriteString(tok.String())
|
||||
} else {
|
||||
out.Write(z.Raw())
|
||||
}
|
||||
}
|
||||
if !markup {
|
||||
return ""
|
||||
}
|
||||
clean := feedHTMLPolicy.Sanitize(out.String())
|
||||
if len(clean) > 128<<10 {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
|
||||
func feedText(raw string, limit int) string {
|
||||
// Text-only reader: no remote images, CSS, script, macros or active HTML.
|
||||
// Plain text for fallback and inert Wiki/task capture.
|
||||
for _, tag := range []string{"</p>", "</div>", "<br>", "<br/>", "<br />", "</li>"} {
|
||||
raw = strings.ReplaceAll(raw, tag, tag+"\n")
|
||||
}
|
||||
@@ -81,6 +137,19 @@ type rssEntry struct {
|
||||
Date string `xml:"pubDate"`
|
||||
Category string `xml:"category"`
|
||||
}
|
||||
type atomText struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
func (v atomText) body() (string, bool) {
|
||||
if v.Type == "xhtml" {
|
||||
return v.Inner, true
|
||||
}
|
||||
return v.Text, v.Type == "html"
|
||||
}
|
||||
|
||||
type atomEntry struct {
|
||||
ID string `xml:"id"`
|
||||
Title string `xml:"title"`
|
||||
@@ -88,13 +157,10 @@ type atomEntry struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content struct {
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
} `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Summary atomText `xml:"summary"`
|
||||
Content atomText `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Category struct {
|
||||
Term string `xml:"term,attr"`
|
||||
} `xml:"category"`
|
||||
@@ -138,6 +204,7 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
}
|
||||
var item store.FeedItem
|
||||
isItem := false
|
||||
htmlBody := false
|
||||
if root == "rss" && t.Name.Local == "item" && depth == 3 {
|
||||
var v rssEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
@@ -150,19 +217,18 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
}
|
||||
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category}
|
||||
isItem = true
|
||||
htmlBody = true
|
||||
} else if root == "feed" && t.Name.Local == "entry" && depth == 2 {
|
||||
var v atomEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
depth--
|
||||
body := v.Content.Text
|
||||
body, isHTML := v.Content.body()
|
||||
if body == "" {
|
||||
body = v.Content.Inner
|
||||
}
|
||||
if body == "" {
|
||||
body = v.Summary
|
||||
body, isHTML = v.Summary.body()
|
||||
}
|
||||
htmlBody = isHTML
|
||||
link := ""
|
||||
for _, l := range v.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
@@ -187,7 +253,13 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
item.Title = feedText(item.Title, 2048)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
if htmlBody {
|
||||
item.BodyHTML = feedHTML(item.Body, base)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
} else {
|
||||
// Atom type=text is literal, even when it contains HTML-looking text.
|
||||
item.Body = feedText(html.EscapeString(item.Body), 32768)
|
||||
}
|
||||
item.Feed = feedText(title, 512)
|
||||
item.Folder = feedText(item.Folder, 512)
|
||||
if item.ID == "" {
|
||||
@@ -314,6 +386,7 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
body = i.Content.Content
|
||||
}
|
||||
item.Body = feedText(body, 32768)
|
||||
item.BodyHTML = feedHTML(body, base)
|
||||
if len(i.Alternate) > 0 {
|
||||
item.URL = feedLink(base, i.Alternate[0].Href)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user