feat: add previewed atomic OPML subscription import and export
Build and publish container images / images (push) Failing after 7m31s

This commit is contained in:
Anya Tokugawa committed 2026-11-08 04:39:54 +09:00
1 parent 38daea737d
commit 7fe64113c7
13 files changed
+782 -8

No files matched your search

+4
View File
@@ -42,6 +42,10 @@ func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
return
}
key := strings.TrimPrefix(r.URL.Path, feedsAPI)
if strings.HasPrefix(key, "opml/") {
s.handleFeedOPML(w, r, domain, actor, key)
return
}
switch {
case key == "note" && r.Method == "POST":
var input struct {
+275
View File
@@ -0,0 +1,275 @@
package server
import (
"encoding/xml"
"io"
"net/http"
"strings"
"pageswiki/internal/store"
)
const feedOPMLMaxBytes = 512 << 10
type feedOPMLRow struct {
store.FeedSubscription
Index int `json:"index"`
Status string `json:"status"`
}
// Parse only the uploaded document. OPML includes and HTML links are never
// fetched. Every token participates in the depth/outline/DTD checks.
func parseFeedOPML(raw string) ([]feedOPMLRow, error) {
if len(raw) > feedOPMLMaxBytes {
return nil, store.ErrTaskDAVLimit
}
d := xml.NewDecoder(strings.NewReader(strings.TrimPrefix(raw, "\ufeff")))
// JSON and the browser's File.text() carry UTF-8, not the original legacy
// bytes. Reject other XML encodings instead of silently corrupting names.
type frame struct {
name xml.Name
folder string
commented bool
}
stack := []frame{}
rows := []feedOPMLRow{}
root, head, body, outlines := 0, 0, 0, 0
for {
token, err := d.Token()
if err == io.EOF {
break
}
if err != nil {
return nil, store.ErrTaskInvalid
}
switch t := token.(type) {
case xml.Directive:
return nil, store.ErrTaskInvalid
case xml.CharData:
if len(stack) == 0 && strings.TrimSpace(string(t)) != "" {
return nil, store.ErrTaskInvalid
}
case xml.StartElement:
if len(stack) >= 16 {
return nil, store.ErrTaskDAVLimit
}
attrs := map[string]string{}
seen := map[xml.Name]bool{}
for _, a := range t.Attr {
if seen[a.Name] {
return nil, store.ErrTaskInvalid
}
seen[a.Name] = true
if a.Name.Space == "" {
attrs[a.Name.Local] = a.Value
}
}
f := frame{name: t.Name}
if len(stack) == 0 {
root++
if root != 1 || t.Name.Space != "" || t.Name.Local != "opml" || (attrs["version"] != "1.0" && attrs["version"] != "1.1" && attrs["version"] != "2.0") {
return nil, store.ErrTaskInvalid
}
} else {
parent := stack[len(stack)-1]
f.folder = parent.folder
f.commented = parent.commented
if len(stack) == 1 && t.Name.Space == "" {
switch t.Name.Local {
case "head":
head++
if head != 1 || body != 0 {
return nil, store.ErrTaskInvalid
}
case "body":
body++
if head != 1 || body != 1 {
return nil, store.ErrTaskInvalid
}
default:
return nil, store.ErrTaskInvalid
}
}
inBody := len(stack) >= 2 && stack[1].name == (xml.Name{Local: "body"})
if inBody && t.Name.Space == "" && parent.name.Space == "" && (parent.name.Local == "body" || parent.name.Local == "outline") && t.Name.Local != "outline" {
return nil, store.ErrTaskInvalid
}
if inBody && t.Name.Space == "" && t.Name.Local == "outline" && parent.name.Space == "" && (parent.name.Local == "body" || parent.name.Local == "outline") {
outlines++
if outlines > 1000 {
return nil, store.ErrTaskDAVLimit
}
f.commented = f.commented || strings.EqualFold(attrs["isComment"], "true")
label := attrs["text"]
if label == "" {
label = attrs["title"]
}
label = feedText(label, 1024)
kind := strings.ToLower(attrs["type"])
if attrs["xmlUrl"] != "" || kind != "" {
if len(rows) >= 500 {
return nil, store.ErrTaskDAVLimit
}
row := feedOPMLRow{FeedSubscription: store.FeedSubscription{Title: label, Folder: f.folder, URL: strings.TrimSpace(attrs["xmlUrl"])}, Index: len(rows), Status: "add"}
switch {
case f.commented:
row.Status = "commented"
case kind != "" && kind != "rss" && kind != "atom":
row.Status = "unsupported_type"
case !validCalendarSourceURL(row.URL):
row.Status = "unsupported_url"
case label == "" || len(label) > 200 || len(f.folder) > 200:
row.Status = "invalid_fields"
}
rows = append(rows, row)
} else if label != "" {
if f.folder != "" {
f.folder += " / "
}
f.folder += label
}
}
}
stack = append(stack, f)
case xml.EndElement:
if len(stack) == 0 {
return nil, store.ErrTaskInvalid
}
stack = stack[:len(stack)-1]
}
}
if root != 1 || head != 1 || body != 1 || len(stack) != 0 {
return nil, store.ErrTaskInvalid
}
return rows, nil
}
func classifyFeedOPML(rows []feedOPMLRow, sources []store.FeedSource) {
existing := map[string]bool{}
seen := map[string]bool{}
for _, s := range sources {
if s.Provider == "local" {
existing[store.FeedURLKey(s.URL)] = true
}
}
for i := range rows {
row := &rows[i]
if row.Status != "add" {
continue
}
key := store.FeedURLKey(row.URL)
if existing[key] {
row.Status = "existing"
} else if seen[key] {
row.Status = "duplicate"
}
seen[key] = true
}
}
type feedOPMLOutline struct {
Text string `xml:"text,attr"`
Title string `xml:"title,attr,omitempty"`
Type string `xml:"type,attr,omitempty"`
URL string `xml:"xmlUrl,attr,omitempty"`
Children []feedOPMLOutline `xml:"outline"`
}
func encodeFeedOPML(sources []store.FeedSource) ([]byte, error) {
var doc struct {
XMLName xml.Name `xml:"opml"`
Version string `xml:"version,attr"`
Title string `xml:"head>title"`
Outlines []feedOPMLOutline `xml:"body>outline"`
}
doc.Version = "2.0"
doc.Title = "PagesWiki RSS"
groups := map[string]int{}
for _, source := range sources {
if source.Provider != "local" {
continue
}
entry := feedOPMLOutline{Text: source.Title, Title: source.Title, Type: "rss", URL: source.URL}
if source.Folder == "" {
doc.Outlines = append(doc.Outlines, entry)
continue
}
i, ok := groups[source.Folder]
if !ok {
i = len(doc.Outlines)
groups[source.Folder] = i
doc.Outlines = append(doc.Outlines, feedOPMLOutline{Text: source.Folder})
}
doc.Outlines[i].Children = append(doc.Outlines[i].Children, entry)
}
raw, err := xml.MarshalIndent(doc, "", " ")
return append([]byte(xml.Header), raw...), err
}
func (s *Server) handleFeedOPML(w http.ResponseWriter, r *http.Request, domain, actor, key string) {
if key == "opml/export" && r.Method == http.MethodGet {
sources, err := s.store.FeedSources(r.Context(), domain, actor)
if err != nil {
taskError(w, err)
return
}
raw, err := encodeFeedOPML(sources)
if err != nil {
taskError(w, err)
return
}
w.Header().Set("Content-Type", "text/x-opml; charset=utf-8")
w.Header().Set("Content-Disposition", `attachment; filename="pageswiki-feeds.opml"`)
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Write(raw)
return
}
if r.Method != http.MethodPost || (key != "opml/preview" && key != "opml/import") {
http.NotFound(w, r)
return
}
var input struct {
OPML string `json:"opml"`
Selected []int `json:"selected"`
RequestID string `json:"request_id"`
Expected string `json:"expected"`
Enabled bool `json:"enabled"`
Interval int `json:"interval_seconds"`
}
if !taskJSONLimit(w, r, &input, 2<<20) {
return
}
rows, err := parseFeedOPML(input.OPML)
if err != nil {
taskError(w, err)
return
}
if key == "opml/preview" {
sources, err := s.store.FeedSources(r.Context(), domain, actor)
if err != nil {
taskError(w, err)
return
}
classifyFeedOPML(rows, sources)
writeEventJSON(w, 200, map[string]any{"rows": rows, "expected": store.FeedSourcesSnapshot(sources), "available": 32 - len(sources)})
return
}
// Reparse on apply; client-supplied titles, URLs or row eligibility are not trusted.
classifyFeedOPML(rows, nil)
seen := map[int]bool{}
plan := store.FeedImport{RequestID: input.RequestID, Expected: input.Expected, Enabled: input.Enabled, Interval: input.Interval}
for _, index := range input.Selected {
if index < 0 || index >= len(rows) || seen[index] || rows[index].Status != "add" {
taskError(w, store.ErrTaskInvalid)
return
}
seen[index] = true
plan.Entries = append(plan.Entries, rows[index].FeedSubscription)
}
ids, err := s.store.ImportFeedSubscriptions(r.Context(), domain, actor, plan)
if err != nil {
taskError(w, err)
return
}
writeEventJSON(w, 201, map[string]any{"source_ids": ids})
}
+240
View File
@@ -0,0 +1,240 @@
package server
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"github.com/google/uuid"
"pageswiki/internal/store"
)
func opmlDoc(outlines string) string {
return `<?xml version="1.0"?><opml version="2.0"><head><title>Test</title></head><body>` + outlines + `</body></opml>`
}
func opmlEntry(name, link string) string {
return `<outline text="` + name + `" type="rss" xmlUrl="` + link + `"/>`
}
func TestFeedOPMLParserAndExport(t *testing.T) {
for _, version := range []string{"1.0", "1.1", "2.0"} {
raw := strings.Replace(opmlDoc(`<outline text="技術"><outline text="Go">`+opmlEntry("A &amp; B", "https://example.org/feed?a=1&amp;b=2")+`</outline></outline>`), `version="2.0"`, `version="`+version+`"`, 1)
rows, err := parseFeedOPML(raw)
if err != nil || len(rows) != 1 || rows[0].Title != "A & B" || rows[0].Folder != "技術 / Go" || rows[0].URL != "https://example.org/feed?a=1&b=2" || rows[0].Status != "add" {
t.Fatal(rows, err)
}
}
for _, bad := range []string{
strings.Replace(opmlDoc(""), `version="1.0"?>`, `version="1.0" encoding="iso-8859-1"?>`, 1),
`<!DOCTYPE opml [<!ENTITY external SYSTEM "file:///etc/passwd">]>` + opmlDoc(""),
opmlDoc("") + opmlDoc(""), opmlDoc("") + "garbage", `<opml version="2.0"><body/></opml>`,
opmlDoc(`<outline text="x" text="y"/>`), opmlDoc(strings.Repeat(`<outline text="deep">`, 20) + strings.Repeat(`</outline>`, 20)),
opmlDoc(strings.Repeat(opmlEntry("x", "https://example.org"), 501)), strings.Repeat("x", feedOPMLMaxBytes+1),
opmlDoc(`<unexpected>` + opmlEntry("x", "https://example.org") + `</unexpected>`),
} {
if _, err := parseFeedOPML(bad); err == nil {
t.Fatal("accepted invalid OPML")
}
}
sources := []store.FeedSource{{Provider: "local", Title: "A & B", Folder: "技術 / Go", URL: "https://example.org/feed?a=1&b=2"}, {Provider: "local", Title: "Paused", Enabled: false, URL: "https://example.org/paused"}, {Provider: "freshrss", Title: "Remote", URL: "https://private.example.org/api/greader.php", Credential: "secret"}}
raw, err := encodeFeedOPML(sources)
if err != nil || strings.Contains(string(raw), "private.example.org") || strings.Contains(string(raw), "secret") {
t.Fatal("export leak", err)
}
rows, err := parseFeedOPML(string(raw))
if err != nil || len(rows) != 2 || rows[0].Folder != "技術 / Go" || rows[0].Title != "A & B" {
t.Fatal("roundtrip", string(raw), rows, err)
}
raw, err = encodeFeedOPML(nil)
if err != nil {
t.Fatal(err)
}
if rows, err = parseFeedOPML(string(raw)); err != nil || len(rows) != 0 {
t.Fatal("empty export", string(raw), err)
}
}
type opmlPreview struct {
Rows []feedOPMLRow `json:"rows"`
Expected string `json:"expected"`
Available int `json:"available"`
}
func previewOPML(t *testing.T, s *Server, token, raw string) opmlPreview {
t.Helper()
w := feedRequestTest(s, token, "POST", "opml/preview", map[string]string{"opml": raw})
var preview opmlPreview
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &preview) != nil {
t.Fatal(w.Code, w.Body)
}
return preview
}
func TestFeedOPMLPreviewImportReplayAndOwnership(t *testing.T) {
s, token, domain, actor, source := feedFixture(t)
raw := opmlDoc(`<outline text="日本語">` + opmlEntry("New", "https://example.org/new") + opmlEntry("Existing", "https://EXAMPLE.org:443/feed.xml") + opmlEntry("Duplicate", "https://example.org/new") + opmlEntry("Private", "https://127.0.0.1/feed") + `<outline text="Skipped" isComment="true">` + opmlEntry("Hidden", "https://example.org/hidden") + `</outline><outline type="include" text="Include" url="https://example.org/external.opml"/>` + opmlEntry("", "https://example.org/empty-title") + `</outline>`)
p := previewOPML(t, s, token, raw)
if p.Available != 31 || len(p.Rows) != 7 {
t.Fatal(p)
}
for i, want := range []string{"add", "existing", "duplicate", "unsupported_url", "commented", "unsupported_type", "invalid_fields"} {
if p.Rows[i].Status != want {
t.Fatal(i, p.Rows[i])
}
}
all, _ := s.store.FeedSources(context.Background(), domain, actor)
if len(all) != 1 {
t.Fatal("preview wrote sources")
}
for _, index := range []int{2, 3, 4, 5, 6} {
w := feedRequestTest(s, token, "POST", "opml/import", map[string]any{"opml": raw, "expected": p.Expected, "selected": []int{index}, "request_id": uuid.NewString(), "interval_seconds": 1800})
if w.Code != 422 {
t.Fatal("ineligible row accepted", index, w.Code)
}
}
input := map[string]any{"opml": raw, "expected": p.Expected, "selected": []int{0}, "request_id": uuid.NewString(), "enabled": false, "interval_seconds": 1800}
w := feedRequestTest(s, token, "POST", "opml/import", input)
if w.Code != 201 {
t.Fatal(w.Code, w.Body)
}
original := w.Body.String()
if retry := feedRequestTest(s, token, "POST", "opml/import", input); retry.Code != 201 || retry.Body.String() != original {
t.Fatal("retry", retry.Code, retry.Body)
}
input["enabled"] = true
if retry := feedRequestTest(s, token, "POST", "opml/import", input); retry.Code != 412 {
t.Fatal("changed retry", retry.Code)
}
input["enabled"] = false
all, _ = s.store.FeedSources(context.Background(), domain, actor)
if len(all) != 2 {
t.Fatal("duplicate import", len(all))
}
for _, v := range all {
if v.UUID != source.UUID {
if v.Enabled || v.Title != "New" || v.Folder != "日本語" {
t.Fatal(v)
}
if err := s.store.FeedSourceAction(context.Background(), domain, actor, v.UUID, "delete", v.Version); err != nil {
t.Fatal(err)
}
}
}
if retry := feedRequestTest(s, token, "POST", "opml/import", input); retry.Code != 201 || retry.Body.String() != original {
t.Fatal("deleted retry", retry.Code, retry.Body)
}
all, _ = s.store.FeedSources(context.Background(), domain, actor)
if len(all) != 1 {
t.Fatal("resurrected source")
}
w = feedRequestTest(s, token, "GET", "opml/export", nil)
if w.Code != 200 || !strings.Contains(w.Header().Get("Content-Disposition"), "attachment") || !strings.Contains(w.Header().Get("Cache-Control"), "no-store") {
t.Fatal(w.Code, w.Header())
}
other, err := s.store.CreatePasswordAccount(domain, "opmlother", "long test password", "Other", "")
if err != nil {
t.Fatal(err)
}
otherToken, err := s.store.CreateAuthSession(domain, other.UUID)
if err != nil {
t.Fatal(err)
}
w = feedRequestTest(s, otherToken, "GET", "opml/export", nil)
if w.Code != 200 || strings.Contains(w.Body.String(), source.URL) {
t.Fatal("owner leak", w.Code, w.Body)
}
if w = feedRequestTest(s, "", "GET", "opml/export", nil); w.Code == 200 {
t.Fatal("anonymous export")
}
if err = s.store.SetPrincipalState(domain, actor, "readonly"); err != nil {
t.Fatal(err)
}
token, err = s.store.CreateAuthSession(domain, actor)
if err != nil {
t.Fatal(err)
}
if w = feedRequestTest(s, token, "GET", "opml/export", nil); w.Code != 200 {
t.Fatal("readonly export", w.Code)
}
if w = feedRequestTest(s, token, "POST", "opml/import", input); w.Code != 403 {
t.Fatal("readonly mutation", w.Code)
}
if err = s.setSystemExtensionEnabled(domain, "organizer.feeds", false); err != nil {
t.Fatal(err)
}
if w = feedRequestTest(s, token, "GET", "opml/export", nil); w.Code != 404 {
t.Fatal("disabled export", w.Code)
}
}
func TestFeedOPMLAtomicityLimitsAndConcurrentApply(t *testing.T) {
s, token, domain, actor, source := feedFixture(t)
raw := opmlDoc(opmlEntry("A", "https://example.org/a") + opmlEntry("B", "https://example.org/b"))
p := previewOPML(t, s, token, raw)
input := map[string]any{"opml": raw, "expected": p.Expected, "selected": []int{0, 1}, "request_id": uuid.NewString(), "interval_seconds": 1800}
if _, err := s.store.DB.Exec(`CREATE TRIGGER fail_feed_import BEFORE INSERT ON feed_import_receipts BEGIN SELECT RAISE(ABORT,'synthetic failure'); END`); err != nil {
t.Fatal(err)
}
if w := feedRequestTest(s, token, "POST", "opml/import", input); w.Code != 500 {
t.Fatal(w.Code, w.Body)
}
all, _ := s.store.FeedSources(context.Background(), domain, actor)
if len(all) != 1 {
t.Fatal("partial source insert")
}
if _, err := s.store.DB.Exec(`DROP TRIGGER fail_feed_import`); err != nil {
t.Fatal(err)
}
// A configuration edit invalidates the preview; neither new source is applied.
source.Title = "Changed"
if _, err := s.store.SaveFeedSource(context.Background(), domain, actor, source); err != nil {
t.Fatal(err)
}
if w := feedRequestTest(s, token, "POST", "opml/import", input); w.Code != 412 {
t.Fatal("stale preview", w.Code)
}
p = previewOPML(t, s, token, raw)
input["expected"] = p.Expected
var wg sync.WaitGroup
codes := make(chan int, 2)
for range 2 {
wg.Add(1)
go func() { defer wg.Done(); codes <- feedRequestTest(s, token, "POST", "opml/import", input).Code }()
}
wg.Wait()
close(codes)
for code := range codes {
if code != 201 {
t.Fatal("concurrent retry", code)
}
}
all, _ = s.store.FeedSources(context.Background(), domain, actor)
if len(all) != 3 {
t.Fatal("duplicate concurrent import")
}
entries := ""
selected := []int{}
for i := 0; i < 32; i++ {
entries += opmlEntry(fmt.Sprint(i), fmt.Sprintf("https://example.org/large/%d", i))
selected = append(selected, i)
}
raw = opmlDoc(entries)
p = previewOPML(t, s, token, raw)
input = map[string]any{"opml": raw, "expected": p.Expected, "selected": selected, "request_id": uuid.NewString(), "interval_seconds": 1800}
if w := feedRequestTest(s, token, "POST", "opml/import", input); w.Code != 507 {
t.Fatal("capacity", w.Code, w.Body)
}
all, _ = s.store.FeedSources(context.Background(), domain, actor)
if len(all) != 3 {
t.Fatal("partial limit import")
}
for _, selection := range [][]int{{-1}, {99}, {0, 0}, {}} {
input["selected"] = selection
if w := feedRequestTest(s, token, "POST", "opml/import", input); w.Code != 422 {
t.Fatal("invalid selection", selection, w.Code)
}
}
}
+8 -1
View File
@@ -19,6 +19,13 @@
d.querySelector('#feed-reader .feed-actions button:last-child').click();await wait(()=>d.querySelector('#feed-reader form'));
const form=d.querySelector('#feed-reader form'),inputs=form.querySelectorAll('input');inputs[2].value='/notes/feed-ui';inputs[2].dispatchEvent(new w.Event('input'));inputs[5].checked=true;form.requestSubmit();await wait(()=>d.querySelector('#feed-reader a[href^="/v/uuid:"]'));
const search=d.querySelector('#feed-toolbar input');search.value='';search.dispatchEvent(new w.Event('input',{bubbles:true}));await wait(()=>d.querySelectorAll('.feed-row').length===100);d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));
assert(d.documentElement.scrollWidth<=w.innerWidth,'Reader overflow');document.body.dataset.vimReady='1';
assert(d.documentElement.scrollWidth<=w.innerWidth,'Reader overflow');
d.getElementById('feed-opml').click();await wait(()=>d.getElementById('feed-opml-file'));
const opml='<opml version="2.0"><head><title>Import</title></head><body><outline text="技術"><outline type="rss" text="Import '+w.innerWidth+'" xmlUrl="https://example.org/ui-'+w.innerWidth+'"/><outline type="rss" text="Existing" xmlUrl="https://example.org/feed.xml"/><outline type="rss" text="Private" xmlUrl="https://127.0.0.1/feed"/></outline></body></opml>';
const choose=()=>{const transfer=new w.DataTransfer();transfer.items.add(new w.File([opml],'feeds.opml',{type:'text/x-opml'}));const f=d.getElementById('feed-opml-file');f.files=transfer.files;f.dispatchEvent(new w.Event('change',{bubbles:true}));};choose();
await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);const checks=d.querySelectorAll('#feed-opml-preview tbody input');assert(checks[0].checked&&!checks[0].disabled&&checks[1].disabled&&checks[2].disabled,'Import eligibility');
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML overflow');d.querySelector('.feed-import-options input[type=checkbox]').checked=false;d.getElementById('feed-opml-preview').requestSubmit();await wait(()=>!d.getElementById('feed-opml-preview'));
assert(d.getElementById('feed-sources').textContent.includes('Import '+w.innerWidth),'Imported source missing');d.getElementById('feed-opml').click();await wait(()=>d.getElementById('feed-opml-file'));choose();await wait(()=>d.querySelectorAll('#feed-opml-preview tbody tr').length===3);assert(d.querySelector('#feed-opml-preview tbody input').disabled,'Duplicate after import');
assert(d.documentElement.scrollWidth<=w.innerWidth,'OPML preview overflow');document.body.dataset.vimReady='1';
}catch(e){const out=document.createElement('pre');out.id='error';out.textContent=e.stack;document.body.append(out);}
})();