feat: add private RSS reader and read-only FreshRSS integration
Build and publish container images / images (push) Failing after 6m19s
Build and publish container images / images (push) Failing after 6m19s
This commit is contained in:
1 parent
0c2cf263b6
commit
0d8997972c
25 files changed
+1911
-14
No files matched your search
@@ -16,6 +16,7 @@ func (s *Server) registeredEventHandlers() *dregarnuhr.Registry {
|
||||
s.eventHandlersOnce.Do(func() {
|
||||
registry := &dregarnuhr.Registry{}
|
||||
handlers := map[string]dregarnuhr.Handler{
|
||||
store.FeedSyncEventType: s.syncFeedSources,
|
||||
store.TaskDigestEventType: s.deliverTaskDigests,
|
||||
store.NotificationPushEventType: func(ctx context.Context, job *store.EventJob) error {
|
||||
return s.deliverNotificationPushContext(ctx, job)
|
||||
|
||||
@@ -58,6 +58,9 @@ func (s *Server) startEventScheduler() {
|
||||
if s.store == nil {
|
||||
return
|
||||
}
|
||||
if err := s.store.EnsureFeedEvent(); err != nil {
|
||||
log.Printf("feed scheduler initialization failed")
|
||||
}
|
||||
if err := s.store.EnsureBuiltinResourceEvent(); err != nil {
|
||||
log.Printf("resource monitor initialization failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
const feedsAPI = "/_/api/v1/extension/organizer.feeds/"
|
||||
|
||||
func feedRoute(p string) bool { return p == "/v/feeds" || strings.HasPrefix(p, feedsAPI) }
|
||||
func (s *Server) handleFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
domain := s.domainUUIDForRequest(r)
|
||||
if !s.systemExtensionEnabled(domain, "organizer.feeds") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
actor, active := s.calendarActor(r)
|
||||
if actor == "" {
|
||||
s.authRequired(w, r)
|
||||
return
|
||||
}
|
||||
if r.Method != "GET" && r.Method != "HEAD" && !active {
|
||||
taskError(w, store.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v/feeds" {
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
w.WriteHeader(405)
|
||||
return
|
||||
}
|
||||
if _, e := s.store.FeedSources(r.Context(), domain, actor); e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
s.renderWithContext(w, r, "feeds.gohtml", map[string]any{"Title": "RSS", "FeedWritable": active})
|
||||
return
|
||||
}
|
||||
key := strings.TrimPrefix(r.URL.Path, feedsAPI)
|
||||
switch {
|
||||
case key == "note" && r.Method == "POST":
|
||||
var input struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Source string `json:"source"`
|
||||
ID string `json:"id"`
|
||||
Path string `json:"path"`
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
Confirm bool `json:"confirm_publication"`
|
||||
}
|
||||
if !taskJSONLimit(w, r, &input, 128<<10) {
|
||||
return
|
||||
}
|
||||
if !input.Confirm {
|
||||
taskError(w, store.ErrTaskInvalid)
|
||||
return
|
||||
}
|
||||
for _, capability := range []string{"read", "create", "write"} {
|
||||
if s.authorizePathRequest(r, capability, input.Path) != nil {
|
||||
taskError(w, store.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
}
|
||||
page, e := s.store.CaptureFeedNote(r.Context(), domain, actor, input.RequestID, input.Source, input.ID, input.Path, input.Title, input.Body)
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 201, map[string]string{"page_uuid": page})
|
||||
return
|
||||
case key == "sources" && r.Method == "GET":
|
||||
sources, e := s.store.FeedSources(r.Context(), domain, actor)
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"sources": sources})
|
||||
return
|
||||
case key == "sources" && r.Method == "POST":
|
||||
var input struct {
|
||||
store.FeedSource
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
if !taskJSON(w, r, &input) {
|
||||
return
|
||||
}
|
||||
if !validCalendarSourceURL(input.URL) || len(input.Username) > 256 || len(input.Password) > 4096 {
|
||||
taskError(w, store.ErrTaskInvalid)
|
||||
return
|
||||
}
|
||||
if input.Provider == "freshrss" {
|
||||
u, _ := url.Parse(input.URL)
|
||||
if u.RawQuery != "" || !strings.HasSuffix(strings.TrimRight(u.Path, "/"), "/greader.php") {
|
||||
taskError(w, store.ErrTaskInvalid)
|
||||
return
|
||||
}
|
||||
}
|
||||
if input.UUID == "" {
|
||||
input.UUID = uuid.NewString()
|
||||
}
|
||||
if input.Version == 0 && input.Provider == "freshrss" {
|
||||
if input.Username == "" || input.Password == "" {
|
||||
taskError(w, store.ErrTaskInvalid)
|
||||
return
|
||||
}
|
||||
cipher, e := s.store.SealFeedCredential(domain, actor, input.UUID, input.Username, input.Password)
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
input.Credential = cipher
|
||||
} else if input.Username != "" || input.Password != "" {
|
||||
taskError(w, store.ErrTaskInvalid)
|
||||
return
|
||||
}
|
||||
v, e := s.store.SaveFeedSource(r.Context(), domain, actor, input.FeedSource)
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"source": v})
|
||||
return
|
||||
case strings.HasPrefix(key, "sources/") && r.Method == "POST":
|
||||
parts := strings.Split(key, "/")
|
||||
if len(parts) != 3 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
var input struct {
|
||||
Version int64 `json:"version"`
|
||||
}
|
||||
if !taskJSON(w, r, &input) {
|
||||
return
|
||||
}
|
||||
if e := s.store.FeedSourceAction(r.Context(), domain, actor, parts[1], parts[2], input.Version); e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]bool{"ok": true})
|
||||
return
|
||||
case key == "items" && r.Method == "GET":
|
||||
q := r.URL.Query()
|
||||
before := int64(-1)
|
||||
if q.Has("before") {
|
||||
var e error
|
||||
before, e = strconv.ParseInt(q.Get("before"), 10, 64)
|
||||
if e != nil || before < 0 {
|
||||
taskError(w, store.ErrTaskInvalid)
|
||||
return
|
||||
}
|
||||
}
|
||||
items, e := s.store.FeedItems(r.Context(), domain, actor, q.Get("source"), q.Get("filter"), q.Get("q"), before, q.Get("key"))
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
more := len(items) > 100
|
||||
if more {
|
||||
items = items[:100]
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"items": items, "more": more})
|
||||
return
|
||||
case key == "item" && r.Method == "GET":
|
||||
v, e := s.store.FeedItem(r.Context(), domain, actor, r.URL.Query().Get("source"), r.URL.Query().Get("id"))
|
||||
if e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"item": v})
|
||||
return
|
||||
case key == "state" && r.Method == "POST":
|
||||
var input struct {
|
||||
Source string `json:"source"`
|
||||
ID string `json:"id"`
|
||||
Read bool `json:"read"`
|
||||
Starred bool `json:"starred"`
|
||||
Version int64 `json:"version"`
|
||||
}
|
||||
if !taskJSON(w, r, &input) {
|
||||
return
|
||||
}
|
||||
if e := s.store.SetFeedState(r.Context(), domain, actor, input.Source, input.ID, input.Read, input.Starred, input.Version); e != nil {
|
||||
taskError(w, e)
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]any{"version": input.Version + 1})
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
func init() {
|
||||
translations["en"]["feeds.title"] = "RSS reader"
|
||||
translations["ja"]["feeds.title"] = "RSSリーダー"
|
||||
translations["en"]["feeds.help"] = "Private local feeds and read-only FreshRSS integration"
|
||||
translations["ja"]["feeds.help"] = "個人用の簡易RSSとFreshRSS読取連携"
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"golang.org/x/net/html/charset"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
var errFeedInvalid = errors.New("invalid_feed")
|
||||
var feedTextPolicy = bluemonday.StrictPolicy()
|
||||
|
||||
func feedText(raw string, limit int) string {
|
||||
// Text-only reader: no remote images, CSS, script, macros or active HTML.
|
||||
for _, tag := range []string{"</p>", "</div>", "<br>", "<br/>", "<br />", "</li>"} {
|
||||
raw = strings.ReplaceAll(raw, tag, tag+"\n")
|
||||
}
|
||||
text := strings.TrimSpace(html.UnescapeString(feedTextPolicy.Sanitize(raw)))
|
||||
if len(text) > limit {
|
||||
text = text[:limit-16]
|
||||
for !utf8.ValidString(text) {
|
||||
text = text[:len(text)-1]
|
||||
}
|
||||
text += "\n[… truncated]"
|
||||
}
|
||||
return text
|
||||
}
|
||||
func feedLink(base, raw string) string {
|
||||
if strings.TrimSpace(raw) == "" || len(raw) > 4096 {
|
||||
return ""
|
||||
}
|
||||
u, e := url.Parse(strings.TrimSpace(raw))
|
||||
if e != nil {
|
||||
return ""
|
||||
}
|
||||
b, e := url.Parse(base)
|
||||
if e != nil {
|
||||
return ""
|
||||
}
|
||||
u = b.ResolveReference(u)
|
||||
if (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil {
|
||||
return ""
|
||||
}
|
||||
result := u.String()
|
||||
if len(result) > 4096 {
|
||||
return ""
|
||||
}
|
||||
return result
|
||||
}
|
||||
func feedDate(v string) int64 {
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, "2006-01-02T15:04:05-0700"} {
|
||||
if t, e := time.Parse(layout, v); e == nil && t.UnixMilli() > 0 && t.Before(time.Now().AddDate(1, 0, 0)) {
|
||||
return t.UnixMilli()
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type rssEntry struct {
|
||||
ID string `xml:"guid"`
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
Content string `xml:"encoded"`
|
||||
Date string `xml:"pubDate"`
|
||||
Category string `xml:"category"`
|
||||
}
|
||||
type atomEntry struct {
|
||||
ID string `xml:"id"`
|
||||
Title string `xml:"title"`
|
||||
Links []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
} `xml:"link"`
|
||||
Summary string `xml:"summary"`
|
||||
Content struct {
|
||||
Text string `xml:",chardata"`
|
||||
Inner string `xml:",innerxml"`
|
||||
} `xml:"content"`
|
||||
Published string `xml:"published"`
|
||||
Updated string `xml:"updated"`
|
||||
Category struct {
|
||||
Term string `xml:"term,attr"`
|
||||
} `xml:"category"`
|
||||
}
|
||||
|
||||
func parseFeed(raw []byte, base string) ([]store.FeedItem, error) {
|
||||
if len(raw) > 4<<20 {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
d := xml.NewDecoder(bytes.NewReader(raw))
|
||||
d.CharsetReader = charset.NewReaderLabel
|
||||
root := ""
|
||||
title := ""
|
||||
depth := 0
|
||||
out := []store.FeedItem{}
|
||||
seen := map[string]bool{}
|
||||
for {
|
||||
tok, e := d.Token()
|
||||
if e == io.EOF {
|
||||
break
|
||||
}
|
||||
if e != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.Directive:
|
||||
return nil, errFeedInvalid // DTD/entities have no role in a feed.
|
||||
case xml.StartElement:
|
||||
depth++
|
||||
if depth == 1 && root != "" {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
if depth > 64 {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
if root == "" {
|
||||
root = t.Name.Local
|
||||
if root != "rss" && root != "feed" {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
}
|
||||
var item store.FeedItem
|
||||
isItem := false
|
||||
if root == "rss" && t.Name.Local == "item" && depth == 3 {
|
||||
var v rssEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
depth--
|
||||
body := v.Content
|
||||
if body == "" {
|
||||
body = v.Description
|
||||
}
|
||||
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: feedLink(base, v.Link), Body: body, PublishedMS: feedDate(v.Date), Folder: v.Category}
|
||||
isItem = true
|
||||
} else if root == "feed" && t.Name.Local == "entry" && depth == 2 {
|
||||
var v atomEntry
|
||||
if d.DecodeElement(&v, &t) != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
depth--
|
||||
body := v.Content.Text
|
||||
if body == "" {
|
||||
body = v.Content.Inner
|
||||
}
|
||||
if body == "" {
|
||||
body = v.Summary
|
||||
}
|
||||
link := ""
|
||||
for _, l := range v.Links {
|
||||
if l.Rel == "" || l.Rel == "alternate" {
|
||||
link = feedLink(base, l.Href)
|
||||
break
|
||||
}
|
||||
}
|
||||
date := v.Published
|
||||
if date == "" {
|
||||
date = v.Updated
|
||||
}
|
||||
item = store.FeedItem{ID: v.ID, Title: v.Title, URL: link, Body: body, PublishedMS: feedDate(date), Folder: v.Category.Term}
|
||||
isItem = true
|
||||
} else if t.Name.Local == "title" && ((root == "rss" && depth == 3) || (root == "feed" && depth == 2)) {
|
||||
if d.DecodeElement(&title, &t) != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
depth--
|
||||
}
|
||||
if isItem {
|
||||
if len(out) >= 500 {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
item.Title = feedText(item.Title, 2048)
|
||||
item.Body = feedText(item.Body, 32768)
|
||||
item.Feed = feedText(title, 512)
|
||||
item.Folder = feedText(item.Folder, 512)
|
||||
if item.ID == "" {
|
||||
key := item.URL
|
||||
if key == "" {
|
||||
key = item.Title + "\x00" + strconv.FormatInt(item.PublishedMS, 10) + "\x00" + item.Body
|
||||
}
|
||||
h := sha256.Sum256([]byte(key))
|
||||
item.ID = "sha256:" + hex.EncodeToString(h[:])
|
||||
}
|
||||
if len(item.ID) > 1024 || seen[item.ID] {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
seen[item.ID] = true
|
||||
out = append(out, item)
|
||||
}
|
||||
case xml.EndElement:
|
||||
depth--
|
||||
}
|
||||
}
|
||||
if root == "" || depth != 0 {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func feedRequest(ctx context.Context, client *http.Client, method, target, auth string, form url.Values, headers map[string]string) ([]byte, http.Header, int, error) {
|
||||
var body io.Reader
|
||||
if form != nil {
|
||||
body = strings.NewReader(form.Encode())
|
||||
}
|
||||
r, e := http.NewRequestWithContext(ctx, method, target, body)
|
||||
if e != nil {
|
||||
return nil, nil, 0, errFeedInvalid
|
||||
}
|
||||
if auth != "" {
|
||||
r.Header.Set("Authorization", "GoogleLogin auth="+auth)
|
||||
}
|
||||
if form != nil {
|
||||
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
}
|
||||
r.Header.Set("User-Agent", "PagesWiki-Feeds/0.1")
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
res, e := client.Do(r)
|
||||
if e != nil {
|
||||
return nil, nil, 0, errors.New("network")
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
return nil, nil, res.StatusCode, errors.New("credentials")
|
||||
}
|
||||
if res.StatusCode == 304 {
|
||||
return nil, res.Header, 304, nil
|
||||
}
|
||||
if res.StatusCode != 200 {
|
||||
return nil, nil, res.StatusCode, errors.New("remote_error")
|
||||
}
|
||||
raw, e := io.ReadAll(io.LimitReader(res.Body, (4<<20)+1))
|
||||
if e != nil {
|
||||
return nil, nil, 0, errors.New("network")
|
||||
}
|
||||
if len(raw) > 4<<20 {
|
||||
return nil, nil, 0, errors.New("limit")
|
||||
}
|
||||
return raw, res.Header, res.StatusCode, nil
|
||||
}
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string) ([]store.FeedItem, error) {
|
||||
base := strings.TrimRight(v.URL, "/")
|
||||
raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
auth := ""
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if strings.HasPrefix(line, "Auth=") {
|
||||
auth = strings.TrimSpace(strings.TrimPrefix(line, "Auth="))
|
||||
}
|
||||
}
|
||||
if auth == "" || len(auth) > 4096 || strings.ContainsAny(auth, "\r\n") {
|
||||
return nil, errors.New("credentials")
|
||||
}
|
||||
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/stream/contents/reading-list?output=json&n=200", auth, nil, nil)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
var response struct {
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Published int64 `json:"published"`
|
||||
Categories []string `json:"categories"`
|
||||
Origin struct {
|
||||
Title string `json:"title"`
|
||||
} `json:"origin"`
|
||||
Alternate []struct {
|
||||
Href string `json:"href"`
|
||||
} `json:"alternate"`
|
||||
Summary struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"summary"`
|
||||
Content struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"content"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if json.Unmarshal(raw, &response) != nil || response.Items == nil || len(response.Items) > 200 {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
out := []store.FeedItem{}
|
||||
seen := map[string]bool{}
|
||||
for _, i := range response.Items {
|
||||
if i.ID == "" || len(i.ID) > 1024 || seen[i.ID] {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
seen[i.ID] = true
|
||||
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512)}
|
||||
if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() {
|
||||
item.PublishedMS = i.Published * 1000
|
||||
}
|
||||
body := i.Summary.Content
|
||||
if body == "" {
|
||||
body = i.Content.Content
|
||||
}
|
||||
item.Body = feedText(body, 32768)
|
||||
if len(i.Alternate) > 0 {
|
||||
item.URL = feedLink(base, i.Alternate[0].Href)
|
||||
}
|
||||
for _, c := range i.Categories {
|
||||
if strings.HasSuffix(c, "/state/com.google/read") {
|
||||
item.Read = true
|
||||
}
|
||||
if strings.HasSuffix(c, "/state/com.google/starred") {
|
||||
item.Starred = true
|
||||
}
|
||||
if pos := strings.Index(c, "/label/"); pos >= 0 {
|
||||
item.Folder = feedText(c[pos+7:], 512)
|
||||
}
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error {
|
||||
client := calendarSourceClient()
|
||||
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
||||
defer client.CloseIdleConnections()
|
||||
return s.syncFeedSourcesWithClient(ctx, job, client)
|
||||
}
|
||||
|
||||
// Tests inject a synthetic transport; production always uses the DNS-pinned,
|
||||
// public HTTPS-only transport above, never a user-configurable HTTP client.
|
||||
func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.EventJob, client *http.Client) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 35*time.Second)
|
||||
defer cancel()
|
||||
sources, e := s.store.DueFeedSources(ctx)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
done, failed := 0, 0
|
||||
for _, v := range sources {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if !s.systemExtensionEnabled(v.Domain, "organizer.feeds") {
|
||||
continue
|
||||
}
|
||||
v, e = s.store.ClaimFeed(ctx, v)
|
||||
if e != nil {
|
||||
continue
|
||||
}
|
||||
child, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var items []store.FeedItem
|
||||
etag, modified := v.ETag, v.Modified
|
||||
unchanged := false
|
||||
if !validCalendarSourceURL(v.URL) {
|
||||
e = errFeedInvalid
|
||||
} else if v.Provider == "freshrss" {
|
||||
var user, password string
|
||||
user, password, e = s.store.OpenFeedCredential(v)
|
||||
if e == nil {
|
||||
items, e = fetchFreshRSS(child, client, v, user, password)
|
||||
} else {
|
||||
e = errors.New("credentials")
|
||||
}
|
||||
} else {
|
||||
var raw []byte
|
||||
var h http.Header
|
||||
var status int
|
||||
raw, h, status, e = feedRequest(child, client, "GET", v.URL, "", nil, map[string]string{"If-None-Match": v.ETag, "If-Modified-Since": v.Modified})
|
||||
if e == nil {
|
||||
unchanged = status == 304
|
||||
if !unchanged {
|
||||
items, e = parseFeed(raw, v.URL)
|
||||
etag, modified = h.Get("ETag"), h.Get("Last-Modified")
|
||||
}
|
||||
}
|
||||
}
|
||||
stop()
|
||||
if e == nil && (len(etag) > 1024 || len(modified) > 1024) {
|
||||
e = errors.New("limit")
|
||||
etag, modified = v.ETag, v.Modified
|
||||
items = nil
|
||||
}
|
||||
code := ""
|
||||
if e != nil {
|
||||
code = e.Error()
|
||||
switch code {
|
||||
case "network", "invalid_feed", "credentials", "remote_error", "limit":
|
||||
default:
|
||||
code = "invalid_feed"
|
||||
}
|
||||
failed++
|
||||
} else {
|
||||
done++
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
|
||||
return e
|
||||
}
|
||||
}
|
||||
s.addEventProgressLog(ctx, job, fmt.Sprintf("feed sources updated=%d failed=%d", done, failed))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
func feedFixture(t *testing.T) (*Server, string, string, string, store.FeedSource) {
|
||||
t.Helper()
|
||||
s, token, domain, actor := taskFixture(t)
|
||||
source, e := s.store.SaveFeedSource(context.Background(), domain, actor, store.FeedSource{UUID: uuid.NewString(), Provider: "local", Title: "Synthetic news", Folder: "Tech", URL: "https://example.org/feed.xml", Enabled: true, Interval: 300})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
sources, e := s.store.FeedSources(context.Background(), domain, actor)
|
||||
if e != nil || len(sources) != 1 {
|
||||
t.Fatal(e)
|
||||
}
|
||||
source = sources[0]
|
||||
return s, token, domain, actor, source
|
||||
}
|
||||
func feedRequestTest(s *Server, token, method, key string, body any) *httptest.ResponseRecorder {
|
||||
raw, _ := json.Marshal(body)
|
||||
r := httptest.NewRequest(method, "http://localhost"+feedsAPI+key, strings.NewReader(string(raw)))
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
if token != "" {
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
s.withAuthorization(s.mux).ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
func TestFeedReaderStateIsolationAndFencing(t *testing.T) {
|
||||
s, token, domain, actor, source := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
claim, e := s.store.ClaimFeed(ctx, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.store.ClaimFeed(ctx, source); !errors.Is(e, store.ErrTaskConflict) {
|
||||
t.Fatal("double claim", e)
|
||||
}
|
||||
item := store.FeedItem{ID: "one", Title: "日本語 news", Body: "safe text", URL: "https://example.org/one", PublishedMS: 1}
|
||||
if e = s.store.CompleteFeed(ctx, claim, []store.FeedItem{item}, "e1", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if w := feedRequestTest(s, token, "GET", "items", nil); w.Code != 200 || !strings.Contains(w.Body.String(), "日本語") {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
if w := feedRequestTest(s, "", "GET", "items", nil); w.Code == 200 {
|
||||
t.Fatal("anonymous")
|
||||
}
|
||||
if _, e = s.store.FeedItem(ctx, domain, uuid.NewString(), source.UUID, "one"); !errors.Is(e, store.ErrAccessDenied) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
state := map[string]any{"source": source.UUID, "id": "one", "read": true, "starred": true, "version": 0}
|
||||
if w := feedRequestTest(s, token, "POST", "state", state); w.Code != 200 {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
if w := feedRequestTest(s, token, "POST", "state", state); w.Code != 412 {
|
||||
t.Fatal("stale state", w.Code, w.Body)
|
||||
}
|
||||
got, e := s.store.FeedItem(ctx, domain, actor, source.UUID, "one")
|
||||
if e != nil || !got.Read || !got.Starred || got.Version != 1 {
|
||||
t.Fatal(got, e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, claim, nil, "", "", "", false); !errors.Is(e, store.ErrTaskConflict) {
|
||||
t.Fatal("old completion", e)
|
||||
}
|
||||
// Re-fetch must preserve local read/star; config edits invalidate in-flight work.
|
||||
source.Version = 1
|
||||
source, e = s.store.SaveFeedSource(ctx, domain, actor, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
sources, _ := s.store.FeedSources(ctx, domain, actor)
|
||||
claim, e = s.store.ClaimFeed(ctx, sources[0])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, claim, []store.FeedItem{item}, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
got, _ = s.store.FeedItem(ctx, domain, actor, source.UUID, "one")
|
||||
if !got.Read || !got.Starred {
|
||||
t.Fatal("lost state")
|
||||
}
|
||||
source, e = s.store.SaveFeedSource(ctx, domain, actor, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
sources, _ = s.store.FeedSources(ctx, domain, actor)
|
||||
claim, e = s.store.ClaimFeed(ctx, sources[0])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
source.Enabled = false
|
||||
source, e = s.store.SaveFeedSource(ctx, domain, actor, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, claim, []store.FeedItem{item}, "", "", "", false); !errors.Is(e, store.ErrTaskConflict) {
|
||||
t.Fatal("disabled late commit", e)
|
||||
}
|
||||
if e = s.setSystemExtensionEnabled(domain, "organizer.feeds", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.store.FeedItems(ctx, domain, actor, "", "", "", -1, ""); !errors.Is(e, store.ErrAccessDenied) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if w := feedRequestTest(s, token, "GET", "items", nil); w.Code != 404 {
|
||||
t.Fatal(w.Code)
|
||||
}
|
||||
}
|
||||
func TestFeedParserSafetyAndFormats(t *testing.T) {
|
||||
rss := `<?xml version="1.0"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/"><channel><title>News</title><item><guid>id</guid><title>日本語</title><link>/one</link><content:encoded><![CDATA[<p>Hello</p><script>alert(1)</script><img src="https://evil.invalid/pixel"><b>world</b>]]></content:encoded><pubDate>Fri, 06 Nov 2026 10:00:00 +0000</pubDate></item></channel></rss>`
|
||||
items, e := parseFeed([]byte(rss), "https://example.org/feed")
|
||||
if e != nil || len(items) != 1 || items[0].URL != "https://example.org/one" || strings.Contains(items[0].Body, "alert") || strings.Contains(items[0].Body, "pixel") || items[0].PublishedMS == 0 {
|
||||
t.Fatal(items, e)
|
||||
}
|
||||
atom := `<feed xmlns="http://www.w3.org/2005/Atom"><title>Atom</title><entry><id>urn:one</id><title>One</title><link href="https://example.org/one"/><updated>2026-11-01T10:00:00Z</updated><content type="html"><p>Body</p></content></entry></feed>`
|
||||
items, e = parseFeed([]byte(atom), "https://example.org")
|
||||
if e != nil || len(items) != 1 || items[0].Body != "Body" {
|
||||
t.Fatal(items, e)
|
||||
}
|
||||
for _, raw := range []string{`<!DOCTYPE rss [<!ENTITY x SYSTEM "file:///etc/passwd">]><rss/>`, `<html>error</html>`, `<rss/><rss/>`, `<rss><channel><item><guid>x</guid></item><item><guid>x</guid></item></channel></rss>`, strings.Repeat("x", (4<<20)+1)} {
|
||||
if _, e = parseFeed([]byte(raw), "https://example.org"); e == nil {
|
||||
t.Fatal("accepted malformed feed")
|
||||
}
|
||||
}
|
||||
if feedLink("https://example.org", "javascript:alert(1)") != "" || feedLink("https://example.org", "https://user:pass@example.org") != "" {
|
||||
t.Fatal("unsafe link")
|
||||
}
|
||||
items, e = parseFeed([]byte(`<rss><channel><item><title>No ID or link</title><description>First body</description></item><item><title>No ID or link</title><description>Second body</description></item></channel></rss>`), "https://example.org/feed")
|
||||
if e != nil || len(items) != 2 || items[0].ID == items[1].ID || items[0].URL != "" {
|
||||
t.Fatal("missing IDs must not collapse to the feed URL", items, e)
|
||||
}
|
||||
}
|
||||
func TestFreshRSSReadOnlyContract(t *testing.T) {
|
||||
requests := []string{}
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests = append(requests, r.Method+" "+r.URL.Path)
|
||||
switch r.URL.Path {
|
||||
case "/api/greader.php/accounts/ClientLogin":
|
||||
if r.Method != "POST" {
|
||||
t.Error("login method")
|
||||
}
|
||||
r.ParseForm()
|
||||
if r.Form.Get("Email") != "reader" || r.Form.Get("Passwd") != "api-secret" {
|
||||
t.Error("login form")
|
||||
}
|
||||
fmt.Fprint(w, "SID=x\nAuth=reader/token\n")
|
||||
case "/api/greader.php/reader/api/0/stream/contents/reading-list":
|
||||
if r.Header.Get("Authorization") != "GoogleLogin auth=reader/token" || r.URL.Query().Get("n") != "200" {
|
||||
t.Error("request contract")
|
||||
}
|
||||
fmt.Fprint(w, `{"items":[{"id":"tag:1","title":"Title","summary":{"content":"<p>Text</p>"},"published":1793500000,"categories":["user/reader/state/com.google/read","user/reader/state/com.google/starred","user/reader/label/Tech"],"origin":{"title":"Feed"},"alternate":[{"href":"https://example.org/article"}]}]}`)
|
||||
default:
|
||||
t.Error("unexpected mutation", r.URL.Path)
|
||||
w.WriteHeader(404)
|
||||
}
|
||||
}))
|
||||
defer remote.Close()
|
||||
items, e := fetchFreshRSS(context.Background(), remote.Client(), store.FeedSource{URL: remote.URL + "/api/greader.php"}, "reader", "api-secret")
|
||||
if e != nil || len(items) != 1 || !items[0].Read || !items[0].Starred || items[0].Folder != "Tech" || items[0].Body != "Text" || len(requests) != 2 {
|
||||
t.Fatal(items, e, requests)
|
||||
}
|
||||
for _, bad := range []string{"http://example.org/feed", "https://127.0.0.1/feed", "https://10.0.0.1/feed", "https://example.org:8443/feed", "https://user:pass@example.org/feed"} {
|
||||
if validCalendarSourceURL(bad) {
|
||||
t.Fatal("SSRF URL", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestFeedPaginationFailureAndDelete(t *testing.T) {
|
||||
s, token, domain, actor, source := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
claim, e := s.store.ClaimFeed(ctx, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items := []store.FeedItem{}
|
||||
for i := 0; i < 205; i++ {
|
||||
items = append(items, store.FeedItem{ID: fmt.Sprintf("id-%03d", i), Title: "Title", Body: "Body", PublishedMS: 100})
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, claim, items, "old", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
before := int64(-1)
|
||||
key := ""
|
||||
for {
|
||||
batch, e := s.store.FeedItems(ctx, domain, actor, "", "", "", before, key)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
more := len(batch) > 100
|
||||
if more {
|
||||
batch = batch[:100]
|
||||
}
|
||||
for _, item := range batch {
|
||||
if seen[item.ID] {
|
||||
t.Fatal("duplicate")
|
||||
}
|
||||
seen[item.ID] = true
|
||||
}
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
last := batch[len(batch)-1]
|
||||
before = last.PublishedMS
|
||||
key = last.Source + ":" + last.ID
|
||||
}
|
||||
if len(seen) != 205 {
|
||||
t.Fatal(len(seen))
|
||||
}
|
||||
source, e = s.store.SaveFeedSource(ctx, domain, actor, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
sources, _ := s.store.FeedSources(ctx, domain, actor)
|
||||
claim, e = s.store.ClaimFeed(ctx, sources[0])
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, claim, nil, "", "", "network", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.store.FeedItem(ctx, domain, actor, source.UUID, "id-001"); e != nil {
|
||||
t.Fatal("cache lost on failure", e)
|
||||
}
|
||||
if w := feedRequestTest(s, token, "POST", "sources/"+source.UUID+"/delete", map[string]int64{"version": source.Version}); w.Code != 200 {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
var count int
|
||||
if e = s.store.DB.QueryRow(`SELECT count(*) FROM feed_items`).Scan(&count); e != nil || count != 0 {
|
||||
t.Fatal("cascade", count, e)
|
||||
}
|
||||
var integrity string
|
||||
if e = s.store.DB.QueryRow(`PRAGMA integrity_check`).Scan(&integrity); e != nil || integrity != "ok" {
|
||||
t.Fatal(integrity, e)
|
||||
}
|
||||
}
|
||||
func TestFeedSourceAPIAndCredentialBinding(t *testing.T) {
|
||||
s, token, domain, actor, _ := feedFixture(t)
|
||||
input := map[string]any{"uuid": uuid.NewString(), "provider": "freshrss", "title": "Remote", "url": "https://rss.example.org/api/greader.php", "interval_seconds": 300, "enabled": true, "username": "reader", "password": "top-secret"}
|
||||
w := feedRequestTest(s, token, "POST", "sources", input)
|
||||
if w.Code != 200 || strings.Contains(w.Body.String(), "secret") {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
// Each HTTP retry seals a different ciphertext, but identical credentials and
|
||||
// request UUID must still return the same source, not create a duplicate.
|
||||
if retry := feedRequestTest(s, token, "POST", "sources", input); retry.Code != 200 || retry.Body.String() != w.Body.String() {
|
||||
t.Fatal("source replay", retry.Code, retry.Body)
|
||||
}
|
||||
input["password"] = "changed-secret"
|
||||
if retry := feedRequestTest(s, token, "POST", "sources", input); retry.Code != 412 {
|
||||
t.Fatal("changed source replay", retry.Code, retry.Body)
|
||||
}
|
||||
input["password"] = "top-secret"
|
||||
sources, e := s.store.FeedSources(context.Background(), domain, actor)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(sources) != 2 {
|
||||
t.Fatal("duplicate source", len(sources))
|
||||
}
|
||||
var remote store.FeedSource
|
||||
for _, v := range sources {
|
||||
if v.Provider == "freshrss" {
|
||||
remote = v
|
||||
}
|
||||
}
|
||||
u, p, e := s.store.OpenFeedCredential(remote)
|
||||
if e != nil || u != "reader" || p != "top-secret" {
|
||||
t.Fatal("credential", e)
|
||||
}
|
||||
remote.Owner = uuid.NewString()
|
||||
if _, _, e = s.store.OpenFeedCredential(remote); e == nil {
|
||||
t.Fatal("credential rebound")
|
||||
}
|
||||
if w = feedRequestTest(s, token, "GET", "sources", nil); w.Code != 200 || strings.Contains(w.Body.String(), "top-secret") || strings.Contains(w.Body.String(), "credential") {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
input["url"] = "https://127.0.0.1/api/greader.php"
|
||||
if w = feedRequestTest(s, token, "POST", "sources", input); w.Code != 422 {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedFreshRSSAuthorityAndRevocation(t *testing.T) {
|
||||
s, token, domain, actor, local := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
v, e := s.store.SaveFeedSource(ctx, domain, actor, store.FeedSource{UUID: uuid.NewString(), Provider: "freshrss", Title: "FreshRSS", URL: "https://example.org/api/greader.php", Enabled: true, Interval: 300})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
claim := func() store.FeedSource {
|
||||
t.Helper()
|
||||
// Simulate the next scheduler tick without a wall-clock wait.
|
||||
if _, err := s.store.DB.Exec(`UPDATE feed_sources SET next_ms=0 WHERE uuid=?`, v.UUID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := s.store.FeedSources(ctx, domain, actor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, source := range all {
|
||||
if source.UUID == v.UUID {
|
||||
v = source
|
||||
}
|
||||
}
|
||||
c, err := s.store.ClaimFeed(ctx, v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
c := claim()
|
||||
if e = s.store.CompleteFeed(ctx, c, []store.FeedItem{{ID: "removed", Title: "Removed"}, {ID: "kept", Title: "Kept", Read: true, Starred: true}}, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.SetFeedState(ctx, domain, actor, v.UUID, "kept", false, false, 0); !errors.Is(e, store.ErrAccessDenied) {
|
||||
t.Fatal("remote state mutation", e)
|
||||
}
|
||||
c = claim()
|
||||
if e = s.store.CompleteFeed(ctx, c, []store.FeedItem{{ID: "kept", Title: "Changed elsewhere", Read: false}}, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items, e := s.store.FeedItems(ctx, domain, actor, v.UUID, "", "", -1, "")
|
||||
if e != nil || len(items) != 1 || items[0].Read || items[0].Starred || items[0].Title != "Changed elsewhere" {
|
||||
t.Fatal("non-authoritative cache", items, e)
|
||||
}
|
||||
c = claim()
|
||||
if e = s.store.SetPrincipalState(domain, actor, "readonly"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// Changing principal state invalidates the previous session.
|
||||
token, e = s.store.CreateAuthSession(domain, actor)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = s.store.FeedItems(ctx, domain, actor, "", "", "", -1, ""); e != nil {
|
||||
t.Fatal("readonly read", e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, c, nil, "", "", "", false); !errors.Is(e, store.ErrAccessDenied) {
|
||||
t.Fatal("late revoked worker", e)
|
||||
}
|
||||
if w := feedRequestTest(s, token, "POST", "sources/"+local.UUID+"/delete", map[string]int64{"version": 1}); w.Code != 403 {
|
||||
t.Fatal("readonly API", w.Code, w.Body)
|
||||
}
|
||||
if due, err := s.store.DueFeedSources(ctx); err != nil || len(due) != 0 {
|
||||
t.Fatal("inactive source starves queue", due, err)
|
||||
}
|
||||
if e = s.store.SetPrincipalState(domain, actor, "active"); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.setSystemExtensionEnabled(domain, "organizer.feeds", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if due, err := s.store.DueFeedSources(ctx); err != nil || len(due) != 0 {
|
||||
t.Fatal("disabled extension starves queue", due, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedNoteAtomicReceiptAndPermissions(t *testing.T) {
|
||||
s, token, domain, _, source := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
claim, e := s.store.ClaimFeed(ctx, source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, claim, []store.FeedItem{{ID: "note", Title: "Title", Body: "Body"}}, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
id := uuid.NewString()
|
||||
input := map[string]any{"request_id": id, "source": source.UUID, "id": "note", "path": "/notes/rss-" + id, "title": "RSS title", "body": "Source: https://example.org\n{{children}} <script>bad()</script>", "confirm_publication": true}
|
||||
w := feedRequestTest(s, token, "POST", "note", input)
|
||||
if w.Code != 201 {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
original := w.Body.String()
|
||||
if w = feedRequestTest(s, token, "POST", "note", input); w.Code != 201 || w.Body.String() != original {
|
||||
t.Fatal("replay", w.Code, w.Body)
|
||||
}
|
||||
var response map[string]string
|
||||
json.Unmarshal(w.Body.Bytes(), &response)
|
||||
rev, e := s.store.GetLatestRevision(response["page_uuid"])
|
||||
if e != nil || strings.Contains(rev.Body, "{{children}}") || strings.Contains(rev.Body, "<script>") {
|
||||
t.Fatal(rev, e)
|
||||
}
|
||||
input["body"] = "changed"
|
||||
if w = feedRequestTest(s, token, "POST", "note", input); w.Code != 412 {
|
||||
t.Fatal("changed replay", w.Code, w.Body)
|
||||
}
|
||||
input["request_id"] = uuid.NewString()
|
||||
if w = feedRequestTest(s, token, "POST", "note", input); w.Code != 409 {
|
||||
t.Fatal("path exists", w.Code, w.Body)
|
||||
}
|
||||
input["path"] = "/u/other/rss"
|
||||
if w = feedRequestTest(s, token, "POST", "note", input); w.Code != 403 {
|
||||
t.Fatal("reserved user space", w.Code, w.Body)
|
||||
}
|
||||
// Roll back page/path when the durable receipt cannot be committed.
|
||||
if _, e = s.store.DB.Exec(`CREATE TRIGGER fail_feed_receipt BEFORE INSERT ON feed_note_receipts BEGIN SELECT RAISE(ABORT,'synthetic failure'); END`); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
input["path"] = "/notes/rollback"
|
||||
if w = feedRequestTest(s, token, "POST", "note", input); w.Code != 500 {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
var n int
|
||||
if e = s.store.DB.QueryRow(`SELECT count(*) FROM paths WHERE domain_uuid=? AND path='/notes/rollback'`, domain).Scan(&n); e != nil || n != 0 {
|
||||
t.Fatal("partial note", n, e)
|
||||
}
|
||||
}
|
||||
|
||||
type feedTransportFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f feedTransportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestFeedBackgroundConditionalFetch(t *testing.T) {
|
||||
s, _, domain, actor, source := feedFixture(t)
|
||||
ctx := context.Background()
|
||||
step := 0
|
||||
client := &http.Client{Transport: feedTransportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
step++
|
||||
if r.URL.String() != source.URL || r.Method != "GET" {
|
||||
t.Fatal("unexpected request", r.Method, r.URL)
|
||||
}
|
||||
deadline, ok := r.Context().Deadline()
|
||||
if !ok || time.Until(deadline) > 8*time.Second {
|
||||
t.Fatal("unbounded fetch")
|
||||
}
|
||||
switch step {
|
||||
case 1:
|
||||
return &http.Response{StatusCode: 200, Header: http.Header{"Etag": {`"feed-1"`}}, Body: io.NopCloser(strings.NewReader(`<rss><channel><item><guid>id</guid><title>First</title></item></channel></rss>`))}, nil
|
||||
case 2:
|
||||
if r.Header.Get("If-None-Match") != `"feed-1"` {
|
||||
t.Fatal("missing conditional GET")
|
||||
}
|
||||
return &http.Response{StatusCode: 304, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(""))}, nil
|
||||
default:
|
||||
return nil, errors.New("synthetic failure containing a secret URL; must not be persisted")
|
||||
}
|
||||
})}
|
||||
for tick := 1; tick <= 3; tick++ {
|
||||
if _, e := s.store.DB.Exec(`UPDATE feed_sources SET next_ms=0 WHERE uuid=?`, source.UUID); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := s.syncFeedSourcesWithClient(ctx, &store.EventJob{UUID: uuid.NewString(), Type: store.FeedSyncEventType}, client); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if step != tick {
|
||||
t.Fatal("wrong fetch count", step, tick)
|
||||
}
|
||||
item, e := s.store.FeedItem(ctx, domain, actor, source.UUID, "id")
|
||||
if e != nil || item.Title != "First" {
|
||||
t.Fatal("cache not preserved", item, e)
|
||||
}
|
||||
all, e := s.store.FeedSources(ctx, domain, actor)
|
||||
if e != nil || len(all) != 1 || all[0].ETag != `"feed-1"` || all[0].Lease != "" || all[0].NextMS <= time.Now().UnixMilli() {
|
||||
t.Fatal("worker metadata", all, e)
|
||||
}
|
||||
if tick == 3 && all[0].Error != "network" {
|
||||
t.Fatal("secret error leaked", all[0].Error)
|
||||
}
|
||||
if e := s.syncFeedSourcesWithClient(ctx, &store.EventJob{UUID: uuid.NewString()}, client); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if step != tick {
|
||||
t.Fatal("cooldown ignored")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedsChromium(t *testing.T) {
|
||||
if os.Getenv("PAGESWIKI_BROWSER_TEST") != "1" {
|
||||
t.Skip("opt-in Chromium")
|
||||
}
|
||||
s, token, domain, actor, source := feedFixture(t)
|
||||
claim, e := s.store.ClaimFeed(context.Background(), source)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
items := []store.FeedItem{}
|
||||
for i := 0; i < 120; i++ {
|
||||
items = append(items, store.FeedItem{ID: fmt.Sprint(i), Title: fmt.Sprintf("%03d 日本語のRSS記事 — リーダーの密度とキーボード操作", i), Body: "安全な本文です。\n\n<script>display as text</script> {{children}}\n\nSource content stays inert.", Feed: "技術ニュース", URL: "https://example.org/article", PublishedMS: int64(1793500000000 + i)})
|
||||
}
|
||||
if e = s.store.CompleteFeed(context.Background(), claim, items, "", "", "", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
_ = domain
|
||||
_ = actor
|
||||
s.store.BackgroundJobsDisabled = true
|
||||
s = New(s.store)
|
||||
script, e := os.ReadFile("internal/server/testdata/feeds.js")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
handler := s.handler()
|
||||
web := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Host = "localhost"
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
if r.URL.Path == "/fixture" {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprint(w, "<!doctype html><body style='margin:0'><script>"+string(script)+"</script></body>")
|
||||
return
|
||||
}
|
||||
handler.ServeHTTP(w, r)
|
||||
}))
|
||||
defer web.Close()
|
||||
for _, width := range []string{"1440", "390"} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
args := []string{"internal/server/testdata/vim-browser-smoke.mjs", web.URL + "/fixture?width=" + url.QueryEscape(width), t.TempDir(), "/tmp/pageswiki-feeds-" + width + ".png", width, "850"}
|
||||
out, e := exec.CommandContext(ctx, "node", args...).CombinedOutput()
|
||||
cancel()
|
||||
if e != nil {
|
||||
t.Fatalf("browser %s: %v %s", width, e, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -93,6 +93,8 @@ func (s *Server) handleDashboardData(w http.ResponseWriter, r *http.Request, act
|
||||
|
||||
func organizerSection(path string) string {
|
||||
switch {
|
||||
case feedRoute(path):
|
||||
return "feeds"
|
||||
case path == "/v/plan/dashboard":
|
||||
return "dashboard"
|
||||
case strings.HasPrefix(path, "/-/sys/config"):
|
||||
|
||||
@@ -68,6 +68,7 @@ func (s *Server) renderWithContext(
|
||||
data["CalendarEnabled"] = s.systemExtensionEnabled(domainUUID, "organizer.calendar")
|
||||
data["TasksEnabled"] = s.systemExtensionEnabled(domainUUID, "organizer.tasks")
|
||||
data["ContactsEnabled"] = s.systemExtensionEnabled(domainUUID, "organizer.contacts")
|
||||
data["FeedsEnabled"] = s.systemExtensionEnabled(domainUUID, "organizer.feeds")
|
||||
data["OrganizerSection"] = organizerSection(r.URL.Path)
|
||||
if r.URL.Path == "/" {
|
||||
data["IsHome"] = true
|
||||
|
||||
@@ -20,6 +20,9 @@ import (
|
||||
const tasksAPI = "/_/api/v1/extension/organizer.tasks/"
|
||||
|
||||
func taskRoute(p string) bool {
|
||||
if feedRoute(p) {
|
||||
return true
|
||||
}
|
||||
if p == "/v/plan/dashboard" {
|
||||
return true
|
||||
}
|
||||
@@ -30,6 +33,8 @@ func taskRoute(p string) bool {
|
||||
}
|
||||
|
||||
func (s *Server) registerTaskRoutes() {
|
||||
s.mux.HandleFunc("/v/feeds", s.handleFeeds)
|
||||
s.mux.HandleFunc(feedsAPI, s.handleFeeds)
|
||||
s.mux.HandleFunc("/v/plan/dashboard", s.handleOrganizerDashboard)
|
||||
s.mux.HandleFunc(contactDAV, s.handleContactDAV)
|
||||
s.mux.HandleFunc(strings.TrimSuffix(contactDAV, "/"), s.handleContactDAV)
|
||||
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
(async()=>{
|
||||
const delay=ms=>new Promise(r=>setTimeout(r,ms)),assert=(ok,msg)=>{if(!ok)throw Error(msg);};
|
||||
const wait=async check=>{for(let i=0;i<200;i++){if(check())return;await delay(30);}throw Error('Timeout: '+check);};
|
||||
const frame=document.createElement('iframe');frame.style='display:block;width:100%;height:100vh;border:0';document.body.append(frame);
|
||||
try{
|
||||
await new Promise(resolve=>{frame.onload=resolve;frame.src='/v/feeds';});const d=frame.contentDocument,w=frame.contentWindow;
|
||||
await wait(()=>d.querySelectorAll('.feed-row').length===100);
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'Viewport overflow');
|
||||
d.getElementById('feed-rows').focus();d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'j',bubbles:true}));
|
||||
await wait(()=>d.querySelector('#feed-reader .feed-body'));assert(d.querySelector('#feed-reader .feed-body').textContent.includes('<script>'),'Text not preserved');assert(!d.querySelector('#feed-reader script'),'Active article HTML');
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'*',bubbles:true}));await wait(()=>d.querySelector('.feed-row[aria-selected=true]').textContent.includes('★'));
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'Escape',bubbles:true}));await wait(()=>!d.querySelector('.feed-layout').classList.contains('reading'));
|
||||
d.getElementById('feed-more').click();await wait(()=>d.querySelectorAll('.feed-row').length===120);
|
||||
d.dispatchEvent(new w.KeyboardEvent('keydown',{key:'/',bubbles:true}));await wait(()=>d.activeElement.type==='search');d.activeElement.value='119';d.activeElement.dispatchEvent(new w.Event('input',{bubbles:true}));await wait(()=>d.querySelectorAll('.feed-row').length===1);
|
||||
d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));
|
||||
d.querySelector('#feed-reader .feed-actions button:last-child').click();await wait(()=>d.querySelector('#feed-reader form'));
|
||||
const form=d.querySelector('#feed-reader form'),inputs=form.querySelectorAll('input');inputs[2].value='/notes/feed-ui';inputs[2].dispatchEvent(new w.Event('input'));inputs[5].checked=true;form.requestSubmit();await wait(()=>d.querySelector('#feed-reader a[href^="/v/uuid:"]'));
|
||||
const search=d.querySelector('#feed-toolbar input');search.value='';search.dispatchEvent(new w.Event('input',{bubbles:true}));await wait(()=>d.querySelectorAll('.feed-row').length===100);d.querySelector('.feed-row').click();await wait(()=>d.querySelector('#feed-reader .feed-body'));
|
||||
assert(d.documentElement.scrollWidth<=w.innerWidth,'Reader overflow');document.body.dataset.vimReady='1';
|
||||
}catch(e){const out=document.createElement('pre');out.id='error';out.textContent=e.stack;document.body.append(out);}
|
||||
})();
|
||||
Reference in new issue
Block a user