feat: render RSS HTML bodies and simplify reader navigation
Build and publish container images / images (push) Successful in 12m32s
Build and publish container images / images (push) Successful in 12m32s
This commit is contained in:
1 parent
0d8997972c
commit
38daea737d
12 files changed
+208
-49
No files matched your search
@@ -172,6 +172,8 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
// Defense in depth for restored/older caches, not just freshly fetched data.
|
||||
v.BodyHTML = feedHTML(v.BodyHTML, v.URL)
|
||||
writeEventJSON(w, 200, map[string]any{"item": v})
|
||||
return
|
||||
case key == "state" && r.Method == "POST":
|
||||
@@ -195,8 +197,8 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
func init() {
|
||||
translations["en"]["feeds.title"] = "RSS reader"
|
||||
translations["ja"]["feeds.title"] = "RSSリーダー"
|
||||
translations["en"]["feeds.title"] = "RSS"
|
||||
translations["ja"]["feeds.title"] = "RSS"
|
||||
translations["en"]["feeds.help"] = "Private local feeds and read-only FreshRSS integration"
|
||||
translations["ja"]["feeds.help"] = "個人用の簡易RSSとFreshRSS読取連携"
|
||||
}
|
||||
@@ -19,15 +19,71 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
xhtml "golang.org/x/net/html"
|
||||
"golang.org/x/net/html/charset"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
var errFeedInvalid = errors.New("invalid_feed")
|
||||
var feedTextPolicy = bluemonday.StrictPolicy()
|
||||
var feedHTMLPolicy = func() *bluemonday.Policy {
|
||||
p := bluemonday.NewPolicy()
|
||||
p.AllowElements("p", "div", "span", "br", "hr", "h1", "h2", "h3", "h4", "h5", "h6", "strong", "b", "em", "i", "u", "s", "del", "ins", "sub", "sup", "mark", "small", "blockquote", "pre", "code", "kbd", "samp", "ul", "ol", "li", "dl", "dt", "dd", "table", "caption", "thead", "tbody", "tfoot", "tr", "th", "td", "a")
|
||||
p.AllowAttrs("href", "target", "rel").OnElements("a")
|
||||
p.AllowAttrs("colspan", "rowspan", "scope").OnElements("th", "td")
|
||||
p.AllowStandardURLs()
|
||||
p.AllowRelativeURLs(false)
|
||||
return p
|
||||
}()
|
||||
|
||||
// Keep article structure without granting fetched HTML script, style, form,
|
||||
// embedding or automatic network capabilities. Wiki capture still uses Body.
|
||||
func feedHTML(raw, base string) string {
|
||||
if raw == "" || len(raw) > 128<<10 {
|
||||
return ""
|
||||
}
|
||||
z := xhtml.NewTokenizer(strings.NewReader(raw))
|
||||
var out strings.Builder
|
||||
markup := false
|
||||
for {
|
||||
typeOf := z.Next()
|
||||
if typeOf == xhtml.ErrorToken {
|
||||
if z.Err() != io.EOF {
|
||||
return ""
|
||||
}
|
||||
break
|
||||
}
|
||||
if typeOf == xhtml.StartTagToken || typeOf == xhtml.SelfClosingTagToken {
|
||||
markup = true
|
||||
tok := z.Token()
|
||||
if tok.Data == "a" {
|
||||
attrs := []xhtml.Attribute{}
|
||||
for _, attr := range tok.Attr {
|
||||
if attr.Key == "href" {
|
||||
if link := feedLink(base, attr.Val); link != "" {
|
||||
attrs = append(attrs, xhtml.Attribute{Key: "href", Val: link})
|
||||
}
|
||||
}
|
||||
}
|
||||
tok.Attr = append(attrs, xhtml.Attribute{Key: "target", Val: "_blank"}, xhtml.Attribute{Key: "rel", Val: "noopener noreferrer"})
|
||||
}
|
||||
out.WriteString(tok.String())
|
||||
} else {
|
||||
out.Write(z.Raw())
|
||||
}
|
||||
}
|
||||
if !markup {
|
||||
return ""
|
||||
}
|
||||
clean := feedHTMLPolicy.Sanitize(out.String())
|
||||
if len(clean) > 128<<10 {
|
||||
return ""
|
||||
}
|
||||
return clean
|
||||
}
|
||||
|
||||
func feedText(raw string, limit int) string {
|
||||
// Text-only reader: no remote images, CSS, script, macros or active HTML.
|
||||
// Plain text for fallback and inert Wiki/task capture.
|
||||
for _, tag := range []string{"</p>", "</div>", "<br>", "<br/>", "<br />", "</li>"} {
|
||||
raw = strings.ReplaceAll(raw, tag, tag+"\n")
|
||||
}
|
||||
@@ -81,6 +137,19 @@ type rssEntry struct {
|
||||
Date string `xml:"pubDate"`
|
||||
Category string `xml:"category"`
|
||||
}
|
||||
type atomText struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
}
|
||||
|
||||
func (v atomText) body() (string, bool) {
|
||||
if v.Type == "xhtml" {
|
||||
return v.Inner, true
|
||||
}
|
||||
return v.Text, v.Type == "html"
|
||||
}
|
||||
|
||||
type atomEntry struct {
|
||||
ID string `xml:"id"`
|
||||
Title string `xml:"title"`
|
||||
@@ -88,13 +157,10 @@ type atomEntry struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content struct {
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
} `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Summary atomText `xml:"summary"`
|
||||
Content atomText `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Category struct {
|
||||
Term string `xml:"term,attr"`
|
||||
} `xml:"category"`
|
||||
@@ -138,6 +204,7 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
}
|
||||
var item store.FeedItem
|
||||
isItem := false
|
||||
htmlBody := false
|
||||
if root == "rss" && t.Name.Local == "item" && depth == 3 {
|
||||
var v rssEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
@@ -150,19 +217,18 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
}
|
||||
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category}
|
||||
isItem = true
|
||||
htmlBody = true
|
||||
} else if root == "feed" && t.Name.Local == "entry" && depth == 2 {
|
||||
var v atomEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
depth--
|
||||
body := v.Content.Text
|
||||
body, isHTML := v.Content.body()
|
||||
if body == "" {
|
||||
body = v.Content.Inner
|
||||
}
|
||||
if body == "" {
|
||||
body = v.Summary
|
||||
body, isHTML = v.Summary.body()
|
||||
}
|
||||
htmlBody = isHTML
|
||||
link := ""
|
||||
for _, l := range v.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
@@ -187,7 +253,13 @@ func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
item.Title = feedText(item.Title, 2048)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
if htmlBody {
|
||||
item.BodyHTML = feedHTML(item.Body, base)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
} else {
|
||||
// Atom type=text is literal, even when it contains HTML-looking text.
|
||||
item.Body = feedText(html.EscapeString(item.Body), 32768)
|
||||
}
|
||||
item.Feed = feedText(title, 512)
|
||||
item.Folder = feedText(item.Folder, 512)
|
||||
if item.ID == "" {
|
||||
@@ -314,6 +386,7 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
body = i.Content.Content
|
||||
}
|
||||
item.Body = feedText(body, 32768)
|
||||
item.BodyHTML = feedHTML(body, base)
|
||||
if len(i.Alternate) > 0 {
|
||||
item.URL = feedLink(base, i.Alternate[0].Href)
|
||||
}
|
||||
|
||||
@@ -176,7 +176,7 @@ func TestFreshRSSReadOnlyContract(t *testing.T) {
|
||||
}))
|
||||
defer remote.Close()
|
||||
items, e := fetchFreshRSS(context.Background(), remote.Client(), store.FeedSource{URL: remote.URL + "/api/greader.php"}, "reader", "api-secret")
|
||||
if e != nil || len(items) != 1 || !items[0].Read || !items[0].Starred || items[0].Folder != "Tech" || items[0].Body != "Text" || len(requests) != 2 {
|
||||
if e != nil || len(items) != 1 || !items[0].Read || !items[0].Starred || items[0].Folder != "Tech" || items[0].Body != "Text" || items[0].BodyHTML != "<p>Text</p>" || len(requests) != 2 {
|
||||
t.Fatal(items, e, requests)
|
||||
}
|
||||
for _, bad := range []string{"http://example.org/feed", "https://127.0.0.1/feed", "https://10.0.0.1/feed", "https://example.org:8443/feed", "https://user:pass@example.org/feed"} {
|
||||
@@ -185,6 +185,45 @@ func TestFreshRSSReadOnlyContract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedHTMLFormattingAndIsolation(t *testing.T) {
|
||||
raw := `<h2>Title</h2><p class="x" style="position:fixed" onclick="bad()"><strong>Bold</strong></p><table><tr><td colspan="2">Cell</td></tr></table><a href="../story" target="_top">Story</a><a href="javascript:bad()">bad</a><script>bad()</script><style>@import '/__feed_leak';</style><form action="/__feed_leak"><input autofocus></form><img src="/__feed_leak"><iframe src="/__feed_leak"></iframe><svg onload="bad()"></svg>`
|
||||
clean := feedHTML(raw, "https://example.org/news/feed.xml")
|
||||
for _, want := range []string{`<h2>Title</h2>`, `<strong>Bold</strong>`, `<table>`, `colspan="2"`, `href="https://example.org/story"`, `target="_blank"`, `noopener`} {
|
||||
if !strings.Contains(clean, want) {
|
||||
t.Fatal("format lost", want, clean)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"<script", "<style", "<form", "<input", "<img", "<iframe", "<svg", "onclick", "javascript:", "position:", "_top", "__feed_leak"} {
|
||||
if strings.Contains(clean, bad) {
|
||||
t.Fatal("unsafe markup", bad, clean)
|
||||
}
|
||||
}
|
||||
if feedHTML(clean, "https://example.org/news/feed.xml") != clean {
|
||||
t.Fatal("sanitizer not stable")
|
||||
}
|
||||
if feedHTML(strings.Repeat("<p>x</p>", 17000), "https://example.org") != "" {
|
||||
t.Fatal("HTML cap")
|
||||
}
|
||||
for _, kind := range []string{"text", "html", "xhtml"} {
|
||||
content := `<b>bold</b>`
|
||||
if kind == "xhtml" {
|
||||
content = `<div xmlns="http://www.w3.org/1999/xhtml"><b>bold</b></div>`
|
||||
}
|
||||
items, e := parseFeed([]byte(`<feed xmlns="http://www.w3.org/2005/Atom"><entry><id>x</id><content type="`+kind+`">`+content+`</content></entry></feed>`), "https://example.org")
|
||||
if e != nil || len(items) != 1 {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if kind == "text" {
|
||||
if items[0].BodyHTML != "" || items[0].Body != "<b>bold</b>" {
|
||||
t.Fatal("literal Atom text", items)
|
||||
}
|
||||
} else if !strings.Contains(items[0].BodyHTML, "<b>bold</b>") {
|
||||
t.Fatal("Atom markup", kind, items)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedPaginationFailureAndDelete(t *testing.T) {
|
||||
s, token, domain, actor, source := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
@@ -500,6 +539,7 @@ func TestFeedsChromium(t *testing.T) {
|
||||
for i := 0; i < 120; i++ {
|
||||
items = append(items, store.FeedItem{ID: fmt.Sprint(i), Title: fmt.Sprintf("%03d 日本語のRSS記事 — リーダーの密度とキーボード操作", i), Body: "安全な本文です。\n\n<script>display as text</script> {{children}}\n\nSource content stays inert.", Feed: "技術ニュース", URL: "https://example.org/article", PublishedMS: int64(1793500000000 + i)})
|
||||
}
|
||||
items[119].BodyHTML = `<h2>HTMLの記事本文</h2><p><strong>強調</strong>や<a href="/article">リンク</a>をそのまま読めます。</p><ul><li>見出しと箇条書き</li><li>テーマに沿った表示</li></ul><table><tr><th>項目</th><th>内容</th></tr><tr><td>本文</td><td>HTML表示</td></tr></table><script>window.rssAttack=1</script><img src="/__feed_leak" onerror="window.rssAttack=1"><style>body{display:none}</style>`
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, items, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
Vendored
+4
-1
@@ -6,8 +6,11 @@
|
||||
await new Promise(resolve=>{frame.onload=resolve;frame.src='/v/feeds';});const d=frame.contentDocument,w=frame.contentWindow;
|
||||
await wait(()=>d.querySelectorAll('.feed-row').length===100);
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'Viewport overflow');
|
||||
assert(d.title.startsWith('RSS ·'),'RSS title');assert(d.getElementById('feed-status').textContent==='','Unrequested status copy');assert(w.getComputedStyle(d.getElementById('feed-status')).display==='none','Empty status takes space');
|
||||
for(const a of d.querySelectorAll('.organizer-nav a')){assert(a.getAttribute('aria-label'),'Unnamed navigation');if(w.innerWidth<=700)assert(w.getComputedStyle(a.querySelector('span')).display==='none','Mobile tab label visible');}
|
||||
d.getElementById('feed-rows').focus();d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'j',bubbles:true}));
|
||||
await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Text not preserved');assert(!d.querySelector('#feed-reader script'),'Active article HTML');
|
||||
await wait(()=>d.querySelector('#feed-reader .feed-body'));const html=d.querySelector('#feed-reader .feed-body').shadowRoot;assert(html?.querySelector('h2')&&html.querySelector('table')&&html.querySelector('strong'),'HTML not rendered');assert(!html.querySelector('script,img,iframe,form'),'Active article HTML');assert(!w.rssAttack,'Script executed');assert(html.querySelector('a').target==='_blank','Article link target');assert(d.querySelector('.feed-original svg')&&!d.querySelector('.feed-original').textContent,'Original link not icon');
|
||||
d.querySelectorAll('.feed-row')[1].click();await wait(()=>d.querySelector('#feed-reader .feed-body')&&!d.querySelector('#feed-reader .feed-body').shadowRoot);assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Literal text not preserved');d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body')?.shadowRoot);
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'*',bubbles:true}));await wait(()=>d.querySelector('.feed-row[aria-selected=true]').textContent.includes('★'));
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));await wait(()=>!d.querySelector('.feed-layout').classList.contains('reading'));
|
||||
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===120);
|
||||
|
||||
+12
-4
@@ -26,7 +26,14 @@ var feedMigrations = []db.SQLMigration{{Version: 1, Name: "private_feed_reader",
|
||||
`CREATE TABLE feed_states(source_uuid TEXT NOT NULL REFERENCES feed_sources(uuid) ON DELETE CASCADE,id TEXT NOT NULL,is_read INTEGER NOT NULL,starred INTEGER NOT NULL,version INTEGER NOT NULL,PRIMARY KEY(source_uuid,id))`,
|
||||
`CREATE TRIGGER feed_source_delete AFTER DELETE ON feed_sources BEGIN DELETE FROM feed_items WHERE source_uuid=old.uuid; DELETE FROM feed_states WHERE source_uuid=old.uuid; END`,
|
||||
`CREATE TABLE feed_note_receipts(domain_uuid TEXT NOT NULL,owner_uuid TEXT NOT NULL,request_id TEXT NOT NULL,input_hash TEXT NOT NULL,page_uuid TEXT NOT NULL,PRIMARY KEY(domain_uuid,owner_uuid,request_id))`,
|
||||
}, Assertions: []string{schemaColumns("feed_sources", "uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified"), schemaColumns("feed_items", "source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred"), schemaUnique("feed_items", "source_uuid,id"), schemaUnique("feed_states", "source_uuid,id"), schemaUnique("feed_note_receipts", "domain_uuid,owner_uuid,request_id"), `SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='feed_source_delete')`}}}
|
||||
}, Assertions: []string{schemaColumns("feed_sources", "uuid,domain_uuid,owner_uuid,provider,title,folder,url,credential,version,enabled,interval_seconds,next_ms,lease,lease_until,last_ms,error,etag,modified"), schemaColumns("feed_items", "source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred"), schemaUnique("feed_items", "source_uuid,id"), schemaUnique("feed_states", "source_uuid,id"), schemaUnique("feed_note_receipts", "domain_uuid,owner_uuid,request_id"), `SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='feed_source_delete')`}}, {
|
||||
Version: 2, Name: "feed_html_body", Statements: []string{
|
||||
`ALTER TABLE feed_items ADD COLUMN body_html TEXT NOT NULL DEFAULT ''`,
|
||||
// Text-only caches cannot recover markup from a 304. Request one full
|
||||
// background fetch while fencing workers started before this migration.
|
||||
`UPDATE feed_sources SET etag='',modified='',next_ms=0,lease='',lease_until=0,version=version+1`,
|
||||
}, Assertions: []string{schemaColumns("feed_items", "body_html")},
|
||||
}}
|
||||
|
||||
// CaptureFeedNote creates an inert Wiki note and receipt atomically. It does not
|
||||
// mutate the feed or create a task, and rechecks grants under the writer lock.
|
||||
@@ -118,6 +125,7 @@ type FeedItem struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
Body string `json:"body,omitempty"`
|
||||
BodyHTML string `json:"body_html,omitempty"`
|
||||
PublishedMS int64 `json:"published_ms"`
|
||||
Feed string `json:"feed"`
|
||||
Folder string `json:"folder"`
|
||||
@@ -309,7 +317,7 @@ func (s *Store) FeedItem(ctx context.Context, domain, owner, source, id string)
|
||||
if e := feedActor(ctx, s.DB, domain, owner, false); e != nil {
|
||||
return v, e
|
||||
}
|
||||
e := s.DB.QueryRowContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.body,i.published_ms,i.feed,i.folder,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND s.uuid=? AND i.id=?`, domain, owner, source, id).Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.Body, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider)
|
||||
e := s.DB.QueryRowContext(ctx, `SELECT i.source_uuid,i.id,i.title,i.url,i.body,i.body_html,i.published_ms,i.feed,i.folder,CASE WHEN s.provider='local' THEN COALESCE(t.is_read,0) ELSE i.is_read END,CASE WHEN s.provider='local' THEN COALESCE(t.starred,0) ELSE i.starred END,COALESCE(t.version,0),s.provider FROM feed_items i JOIN feed_sources s ON s.uuid=i.source_uuid LEFT JOIN feed_states t ON t.source_uuid=i.source_uuid AND t.id=i.id WHERE s.domain_uuid=? AND s.owner_uuid=? AND s.uuid=? AND i.id=?`, domain, owner, source, id).Scan(&v.Source, &v.ID, &v.Title, &v.URL, &v.Body, &v.BodyHTML, &v.PublishedMS, &v.Feed, &v.Folder, &v.Read, &v.Starred, &v.Version, &v.Provider)
|
||||
return v, e
|
||||
}
|
||||
func (s *Store) SetFeedState(ctx context.Context, domain, owner, source, id string, read, star bool, version int64) error {
|
||||
@@ -416,7 +424,7 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
for _, i := range items {
|
||||
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
|
||||
if i.ID == "" || len(i.ID) > 1024 || len(i.Title) > 2048 || len(i.URL) > 4096 || len(i.Body) > 32768 || len(i.BodyHTML) > 128<<10 || len(i.Feed) > 512 || len(i.Folder) > 512 || i.PublishedMS < 0 {
|
||||
return ErrTaskInvalid
|
||||
}
|
||||
}
|
||||
@@ -440,7 +448,7 @@ func (s *Store) CompleteFeed(ctx context.Context, v FeedSource, items []FeedItem
|
||||
}
|
||||
}
|
||||
for _, i := range items {
|
||||
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,published_ms,feed,folder,is_read,starred) VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred)
|
||||
_, e = tx.ExecContext(ctx, `INSERT INTO feed_items(source_uuid,id,title,url,body,body_html,published_ms,feed,folder,is_read,starred) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_uuid,id) DO UPDATE SET title=excluded.title,url=excluded.url,body=excluded.body,body_html=excluded.body_html,published_ms=excluded.published_ms,feed=excluded.feed,folder=excluded.folder,is_read=excluded.is_read,starred=excluded.starred`, v.UUID, i.ID, i.Title, i.URL, i.Body, i.BodyHTML, i.PublishedMS, i.Feed, i.Folder, i.Read, i.Starred)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -20,17 +20,41 @@ func TestFeedMigrationAtomicRepeatAndCleanup(t *testing.T) {
|
||||
if e := st.DB.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='feed_sources'`).Scan(&count); e != nil || count != 0 {
|
||||
t.Fatal("partial schema", count, e)
|
||||
}
|
||||
if e := st.ensureSchemaComponent("feeds"); e != nil {
|
||||
// Upgrade an existing v1 cache, not just an empty database.
|
||||
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", feedMigrations[:1]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := st.DB.Exec(`INSERT INTO feed_sources(uuid,domain_uuid,owner_uuid,provider,title,folder,url,version,enabled,interval_seconds) VALUES('s','d','u','local','title','','https://example.org',1,1,300); INSERT INTO feed_items VALUES('s','i','title','','body',0,'','',0,0); INSERT INTO feed_states VALUES('s','i',1,1,1)`); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := st.DB.Exec(`UPDATE feed_sources SET etag='old',modified='date',next_ms=999,lease='old-worker',lease_until=999`); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
broken = append([]db.SQLMigration(nil), feedMigrations...)
|
||||
broken[1].Statements = append(append([]string(nil), feedMigrations[1].Statements...), `INSERT INTO no_such_feed_table VALUES(1)`)
|
||||
if e := db.ApplySQLiteMigrations(ctx, st.DB, "feeds", broken); e == nil {
|
||||
t.Fatal("expected v2 failure")
|
||||
}
|
||||
if e := st.DB.QueryRow(`SELECT count(*) FROM pragma_table_info('feed_items') WHERE name='body_html'`).Scan(&count); e != nil || count != 0 {
|
||||
t.Fatal("partial v2 schema", count, e)
|
||||
}
|
||||
var oldETag string
|
||||
if e := st.DB.QueryRow(`SELECT etag FROM feed_sources WHERE uuid='s'`).Scan(&oldETag); e != nil || oldETag != "old" {
|
||||
t.Fatal("partial v2 metadata", oldETag, e)
|
||||
}
|
||||
for range 2 {
|
||||
if e := st.ensureSchemaComponent("feeds"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
var body, markup, etag, lease string
|
||||
var version, next int64
|
||||
if e := st.DB.QueryRow(`SELECT body,body_html FROM feed_items WHERE source_uuid='s'`).Scan(&body, &markup); e != nil || body != "body" || markup != "" {
|
||||
t.Fatal("body migration", body, markup, e)
|
||||
}
|
||||
if e := st.DB.QueryRow(`SELECT version,next_ms,etag,lease FROM feed_sources WHERE uuid='s'`).Scan(&version, &next, &etag, &lease); e != nil || version != 2 || next != 0 || etag != "" || lease != "" {
|
||||
t.Fatal("refetch/fence", version, next, etag, lease, e)
|
||||
}
|
||||
if e := st.DB.QueryRow(`SELECT count(*) FROM feed_items`).Scan(&count); e != nil || count != 1 {
|
||||
t.Fatal("lost data", e)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user