feat(feeds): cache FreshRSS subscriptions and category navigation
Build and publish container images / images (push) Failing after 6m7s
Build and publish container images / images (push) Failing after 6m7s
This commit is contained in:
1 parent
7fe64113c7
commit
bfed3d20f5
12 files changed
+503
-34
No files matched your search
@@ -328,7 +328,7 @@ func feedRequest(ctx context.Context, client *http.Client, method, target, auth
|
||||
}
|
||||
return raw, res.Header, res.StatusCode, nil
|
||||
}
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string) ([]store.FeedItem, error) {
|
||||
func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource, user, password string, catalogs ...*[]store.FeedRemoteSubscription) ([]store.FeedItem, error) {
|
||||
base := strings.TrimRight(v.URL, "/")
|
||||
raw, _, _, e := feedRequest(ctx, client, "POST", base+"/accounts/ClientLogin", "", url.Values{"Email": {user}, "Passwd": {password}}, nil)
|
||||
if e != nil {
|
||||
@@ -354,7 +354,8 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
Published int64 `json:"published"`
|
||||
Categories []string `json:"categories"`
|
||||
Origin struct {
|
||||
Title string `json:"title"`
|
||||
Title string `json:"title"`
|
||||
StreamID string `json:"streamId"`
|
||||
} `json:"origin"`
|
||||
Alternate []struct {
|
||||
Href string `json:"href"`
|
||||
@@ -377,7 +378,10 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
seen[i.ID] = true
|
||||
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512)}
|
||||
if (i.Origin.StreamID != "" && !store.ValidFeedRemoteID(i.Origin.StreamID)) || len(i.Categories) > 64 {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
item := store.FeedItem{ID: i.ID, Title: feedText(i.Title, 2048), Feed: feedText(i.Origin.Title, 512), RemoteFeed: i.Origin.StreamID, RemoteCategories: []string{}}
|
||||
if i.Published > 0 && i.Published < time.Now().AddDate(1, 0, 0).Unix() {
|
||||
item.PublishedMS = i.Published * 1000
|
||||
}
|
||||
@@ -391,6 +395,9 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
item.URL = feedLink(base, i.Alternate[0].Href)
|
||||
}
|
||||
for _, c := range i.Categories {
|
||||
if !store.ValidFeedRemoteID(c) {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
if strings.HasSuffix(c, "/state/com.google/read") {
|
||||
item.Read = true
|
||||
}
|
||||
@@ -399,12 +406,53 @@ func fetchFreshRSS(ctx context.Context, client *http.Client, v store.FeedSource,
|
||||
}
|
||||
if pos := strings.Index(c, "/label/"); pos >= 0 {
|
||||
item.Folder = feedText(c[pos+7:], 512)
|
||||
item.RemoteCategories = append(item.RemoteCategories, c)
|
||||
}
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if len(catalogs) > 0 {
|
||||
raw, _, _, e = feedRequest(ctx, client, "GET", base+"/reader/api/0/subscription/list?output=json", auth, nil, nil)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
catalog, e := parseFreshRSSCatalog(raw)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
*catalogs[0] = catalog
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseFreshRSSCatalog(raw []byte) ([]store.FeedRemoteSubscription, error) {
|
||||
if len(raw) > 4<<20 || !utf8.Valid(raw) {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
var response struct {
|
||||
Subscriptions []store.FeedRemoteSubscription `json:"subscriptions"`
|
||||
}
|
||||
if json.Unmarshal(raw, &response) != nil || response.Subscriptions == nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
if len(response.Subscriptions) > 1000 {
|
||||
return nil, errors.New("limit")
|
||||
}
|
||||
for i := range response.Subscriptions {
|
||||
sub := &response.Subscriptions[i]
|
||||
sub.Title = feedText(sub.Title, 512)
|
||||
if sub.Categories == nil {
|
||||
sub.Categories = []store.FeedCategory{}
|
||||
}
|
||||
for j := range sub.Categories {
|
||||
sub.Categories[j].Label = feedText(sub.Categories[j].Label, 512)
|
||||
}
|
||||
}
|
||||
if _, e := store.FeedCatalogJSON(response.Subscriptions); e != nil {
|
||||
return nil, errFeedInvalid
|
||||
}
|
||||
return response.Subscriptions, nil
|
||||
}
|
||||
func (s *Server) syncFeedSources(ctx context.Context, job *store.EventJob) error {
|
||||
client := calendarSourceClient()
|
||||
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
||||
@@ -435,6 +483,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
}
|
||||
child, stop := context.WithTimeout(ctx, 8*time.Second)
|
||||
var items []store.FeedItem
|
||||
var catalog []store.FeedRemoteSubscription
|
||||
etag, modified := v.ETag, v.Modified
|
||||
unchanged := false
|
||||
if !validCalendarSourceURL(v.URL) {
|
||||
@@ -443,7 +492,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
var user, password string
|
||||
user, password, e = s.store.OpenFeedCredential(v)
|
||||
if e == nil {
|
||||
items, e = fetchFreshRSS(child, client, v, user, password)
|
||||
items, e = fetchFreshRSS(child, client, v, user, password, &catalog)
|
||||
} else {
|
||||
e = errors.New("credentials")
|
||||
}
|
||||
@@ -478,7 +527,7 @@ func (s *Server) syncFeedSourcesWithClient(ctx context.Context, job *store.Event
|
||||
} else {
|
||||
done++
|
||||
}
|
||||
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
|
||||
if e = s.store.CompleteFeed(ctx, v, items, etag, modified, code, unchanged, catalog); e != nil && !errors.Is(e, store.ErrAccessDenied) && !errors.Is(e, store.ErrTaskConflict) {
|
||||
return e
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user