feat: render RSS HTML bodies and simplify reader navigation
Build and publish container images / images (push) Successful in 12m32s
Build and publish container images / images (push) Successful in 12m32s
This commit is contained in:
1 parent
0d8997972c
commit
38daea737d
12 files changed
+208
-49
No files matched your search
@@ -172,6 +172,8 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
// Defense in depth for restored/older caches, not just freshly fetched data.
|
||||
v.BodyHTML = feedHTML(v.BodyHTML, v.URL)
|
||||
writeEventJSON(w, 200, map[string]any{"item": v})
|
||||
return
|
||||
case key == "state" && r.Method == "POST":
|
||||
@@ -195,8 +197,8 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
func init() {
|
||||
translations["en"]["feeds.title"] = "RSS reader"
|
||||
translations["ja"]["feeds.title"] = "RSSリーダー"
|
||||
translations["en"]["feeds.title"] = "RSS"
|
||||
translations["ja"]["feeds.title"] = "RSS"
|
||||
translations["en"]["feeds.help"] = "Private local feeds and read-only FreshRSS integration"
|
||||
translations["ja"]["feeds.help"] = "個人用の簡易RSSとFreshRSS読取連携"
|
||||
}
|
||||
@@ -19,15 +19,71 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
xhtml "golang.org/x/net/html"
|
||||
"golang.org/x/net/html/charset"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
var errFeedInvalid = errors.New("invalid_feed")
|
||||
var feedTextPolicy = bluemonday.StrictPolicy()
|
||||
var feedHTMLPolicy = func() *bluemonday.Policy {
|
||||
p := bluemonday.NewPolicy()
|
||||
p.AllowElements("p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "b", "em", "i", "u", "s", "del", "ins", "sub", "sup", "mark", "small", "blockquote", "pre", "code", "kbd", "samp", "ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td", "a")
|
||||
p.AllowAttrs("href", "target", "rel").OnElements("a")
|
||||
p.AllowAttrs("colspan", "rowspan", "scope").OnElements("th", "td")
|
||||
p.AllowStandardURLs()
|
||||
p.AllowRelativeURLs(false)
|
||||
return p
|
||||
}()
|
||||
|
||||
// Keep article structure without granting fetched HTML script, style, form,
|
||||
// embedding or automatic network capabilities. Wiki capture still uses Body.
|
||||
func feedHTML(raw, base string) string {
|
||||
if raw == "" || len(raw) > 128<<10 {
|
||||
return ""
|
||||
}
|
||||
z := xhtml.NewTokenizer(strings.NewReader(raw))
|
||||
var out strings.Builder
|
||||
markup := false
|
||||
for {
|
||||
typeOf := z.Next()
|
||||
if typeOf == xhtml.ErrorToken {
|
||||
if z.Err() != io.EOF {
|
||||
return ""
|
||||
}
|
||||
break
|
||||
}
|
||||
if typeOf == xhtml.StartTagToken || typeOf == xhtml.SelfClosingTagToken {
|
||||
markup = true
|
||||
tok := z.Token()
|
||||
if tok.Data == "a" {
|
||||
attrs := []xhtml.Attribute{}
|
||||
for _, attr := range tok.Attr {
|
||||
if attr.Key == "href" {
|
||||
if link := feedLink(base, attr.Val); link != "" {
|
||||
attrs = append(attrs, xhtml.Attribute{Key: "href", Val: link})
|
||||
}
|
||||
}
|
||||
}
|
||||
tok.Attr = append(attrs, xhtml.Attribute{Key: "target", Val: "_blank"}, xhtml.Attribute{Key: "rel", Val: "noopener noreferrer"})
|
||||
}
|
||||
out.WriteString(tok.String())
|
||||
} else {
|
||||
out.Write(z.Raw())
|
||||
}
|
||||
}
|
||||
if !markup {
|
||||
return ""
|
||||
}
|
||||
clean := feedHTMLPolicy.Sanitize(out.String())
|
||||
if len(clean) > 128<<10 {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
|
||||
func feedText(raw string, limit int) string {
|
||||
// Text-only reader: no remote images, CSS, script, macros or active HTML.
|
||||
// Plain text for fallback and inert Wiki/task capture.
|
||||
for _, tag := range []string{"</p>", "</div>", "<br>", "<br/>", "<br />", "</li>"} {
|
||||
raw = strings.ReplaceAll(raw, tag, tag+"\n")
|
||||
}
|
||||
@@ -81,6 +137,19 @@ type rssEntry struct {
|
||||
Date string `xml:"pubDate"`
|
||||
Category string `xml:"category"`
|
||||
}
|
||||
type atomText struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
func (v atomText) body() (string, bool) {
|
||||
if v.Type == "xhtml" {
|
||||
return v.Inner, true
|
||||
}
|
||||
return v.Text, v.Type == "html"
|
||||
}
|
||||
|
||||
type atomEntry struct {
|
||||
ID string `xml:"id"`
|
||||
Title string `xml:"title"`
|
||||
@@ -88,13 +157,10 @@ type atomEntry struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content struct {
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
} `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Summary atomText `xml:"summary"`
|
||||
Content atomText `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Category struct {
|
||||
Term string `xml:"term,attr"`
|
||||
} `xml:"category"`
|
||||
@@ -138,6 +204,7 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
}
|
||||
var item store.FeedItem
|
||||
isItem := false
|
||||
htmlBody := false
|
||||
if root == "rss" && t.Name.Local == "item" && depth == 3 {
|
||||
var v rssEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
@@ -150,19 +217,18 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
}
|
||||
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category}
|
||||
isItem = true
|
||||
htmlBody = true
|
||||
} else if root == "feed" && t.Name.Local == "entry" && depth == 2 {
|
||||
var v atomEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
depth--
|
||||
body := v.Content.Text
|
||||
body, isHTML := v.Content.body()
|
||||
if body == "" {
|
||||
body = v.Content.Inner
|
||||
}
|
||||
if body == "" {
|
||||
body = v.Summary
|
||||
body, isHTML = v.Summary.body()
|
||||
}
|
||||
htmlBody = isHTML
|
||||
link := ""
|
||||
for _, l := range v.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
@@ -187,7 +253,13 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
item.Title = feedText(item.Title, 2048)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
if htmlBody {
|
||||
item.BodyHTML = feedHTML(item.Body, base)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
} else {
|
||||
// Atom type=text is literal, even when it contains HTML-looking text.
|
||||
item.Body = feedText(html.EscapeString(item.Body), 32768)
|
||||
}
|
||||
item.Feed = feedText(title, 512)
|
||||
item.Folder = feedText(item.Folder, 512)
|
||||
if item.ID == "" {
|
||||
@@ -314,6 +386,7 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
body = i.Content.Content
|
||||
}
|
||||
item.Body = feedText(body, 32768)
|
||||
item.BodyHTML = feedHTML(body, base)
|
||||
if len(i.Alternate) > 0 {
|
||||
item.URL = feedLink(base, i.Alternate[0].Href)
|
||||
}
|
||||
|
||||
@@ -176,7 +176,7 @@ func TestFreshRSSReadOnlyContract(t *testing.T) {
|
||||
}))
|
||||
defer remote.Close()
|
||||
items, e := fetchFreshRSS(context.Background(), remote.Client(), store.FeedSource{URL: remote.URL + "/api/greader.php"}, "reader", "api-secret")
|
||||
if e != nil || len(items) != 1 || !items[0].Read || !items[0].Starred || items[0].Folder != "Tech" || items[0].Body != "Text" || len(requests) != 2 {
|
||||
if e != nil || len(items) != 1 || !items[0].Read || !items[0].Starred || items[0].Folder != "Tech" || items[0].Body != "Text" || items[0].BodyHTML != "<p>Text</p>" || len(requests) != 2 {
|
||||
t.Fatal(items, e, requests)
|
||||
}
|
||||
for _, bad := range []string{"http://example.org/feed", "https://127.0.0.1/feed", "https://10.0.0.1/feed", "https://example.org:8443/feed", "https://user:pass@example.org/feed"} {
|
||||
@@ -185,6 +185,45 @@ func TestFreshRSSReadOnlyContract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedHTMLFormattingAndIsolation(t *testing.T) {
|
||||
raw := `<h2>Title</h2><p class="x" style="position:fixed" onclick="bad()"><strong>Bold</strong></p><table><tr><td colspan="2">Cell</td></tr></table><a href="../story" target="_top">Story</a><a href="javascript:bad()">bad</a><script>bad()</script><style>@import '/__feed_leak';</style><form action="/__feed_leak"><input autofocus></form><img src="/__feed_leak"><iframe src="/__feed_leak"></iframe><svg onload="bad()"></svg>`
|
||||
clean := feedHTML(raw, "https://example.org/news/feed.xml")
|
||||
for _, want := range []string{`<h2>Title</h2>`, `<strong>Bold</strong>`, `<table>`, `colspan="2"`, `href="https://example.org/story"`, `target="_blank"`, `noopener`} {
|
||||
if !strings.Contains(clean, want) {
|
||||
t.Fatal("format lost", want, clean)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"<script", "<style", "<form", "<input", "<img", "<iframe", "<svg", "onclick", "javascript:", "position:", "_top", "__feed_leak"} {
|
||||
if strings.Contains(clean, bad) {
|
||||
t.Fatal("unsafe markup", bad, clean)
|
||||
}
|
||||
}
|
||||
if feedHTML(clean, "https://example.org/news/feed.xml") != clean {
|
||||
t.Fatal("sanitizer not stable")
|
||||
}
|
||||
if feedHTML(strings.Repeat("<p>x</p>", 17000), "https://example.org") != "" {
|
||||
t.Fatal("HTML cap")
|
||||
}
|
||||
for _, kind := range []string{"text", "html", "xhtml"} {
|
||||
content := `<b>bold</b>`
|
||||
if kind == "xhtml" {
|
||||
content = `<div xmlns="http://www.w3.org/1999/xhtml"><b>bold</b></div>`
|
||||
}
|
||||
items, e := parseFeed([]byte(`<feed xmlns="http://www.w3.org/2005/Atom"><entry><id>x</id><content type="`+kind+`">`+content+`</content></entry></feed>`), "https://example.org")
|
||||
if e != nil || len(items) != 1 {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if kind == "text" {
|
||||
if items[0].BodyHTML != "" || items[0].Body != "<b>bold</b>" {
|
||||
t.Fatal("literal Atom text", items)
|
||||
}
|
||||
} else if !strings.Contains(items[0].BodyHTML, "<b>bold</b>") {
|
||||
t.Fatal("Atom markup", kind, items)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedPaginationFailureAndDelete(t *testing.T) {
|
||||
s, token, domain, actor, source := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
@@ -500,6 +539,7 @@ func TestFeedsChromium(t *testing.T) {
|
||||
for i := 0; i < 120; i++ {
|
||||
items = append(items, store.FeedItem{ID: fmt.Sprint(i), Title: fmt.Sprintf("%03d 日本語のRSS記事 — リーダーの密度とキーボード操作", i), Body: "安全な本文です。\n\n<script>display as text</script> {{children}}\n\nSource content stays inert.", Feed: "技術ニュース", URL: "https://example.org/article", PublishedMS: int64(1793500000000 + i)})
|
||||
}
|
||||
items[119].BodyHTML = `<h2>HTMLの記事本文</h2><p><strong>強調</strong>や<a href="/article">リンク</a>をそのまま読めます。</p><ul><li>見出しと箇条書き</li><li>テーマに沿った表示</li></ul><table><tr><th>項目</th><th>内容</th></tr><tr><td>本文</td><td>HTML表示</td></tr></table><script>window.rssAttack=1</script><img src="/__feed_leak" onerror="window.rssAttack=1"><style>body{display:none}</style>`
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, items, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
Vendored
+4
-1
@@ -6,8 +6,11 @@
|
||||
await new Promise(resolve=>{frame.onload=resolve;frame.src='/v/feeds';});const d=frame.contentDocument,w=frame.contentWindow;
|
||||
await wait(()=>d.querySelectorAll('.feed-row').length===100);
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'Viewport overflow');
|
||||
assert(d.title.startsWith('RSS ·'),'RSS title');assert(d.getElementById('feed-status').textContent==='','Unrequested status copy');assert(w.getComputedStyle(d.getElementById('feed-status')).display==='none','Empty status takes space');
|
||||
for(const a of d.querySelectorAll('.organizer-nav a')){assert(a.getAttribute('aria-label'),'Unnamed navigation');if(w.innerWidth<=700)assert(w.getComputedStyle(a.querySelector('span')).display==='none','Mobile tab label visible');}
|
||||
d.getElementById('feed-rows').focus();d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'j',bubbles:true}));
|
||||
await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Text not preserved');assert(!d.querySelector('#feed-reader script'),'Active article HTML');
|
||||
await wait(()=>d.querySelector('#feed-reader .feed-body'));const html=d.querySelector('#feed-reader .feed-body').shadowRoot;assert(html?.querySelector('h2')&&html.querySelector('table')&&html.querySelector('strong'),'HTML not rendered');assert(!html.querySelector('script,img,iframe,form'),'Active article HTML');assert(!w.rssAttack,'Script executed');assert(html.querySelector('a').target==='_blank','Article link target');assert(d.querySelector('.feed-original svg')&&!d.querySelector('.feed-original').textContent,'Original link not icon');
|
||||
d.querySelectorAll('.feed-row')[1].click();await wait(()=>d.querySelector('#feed-reader .feed-body')&&!d.querySelector('#feed-reader .feed-body').shadowRoot);assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Literal text not preserved');d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body')?.shadowRoot);
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'*',bubbles:true}));await wait(()=>d.querySelector('.feed-row[aria-selected=true]').textContent.includes('★'));
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));await wait(()=>!d.querySelector('.feed-layout').classList.contains('reading'));
|
||||
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===120);
|
||||
|
||||
Reference in new issue
Block a user