feat: support mailing-list and monthly mail capture paths

This commit is contained in:
Anya Tokugawa committed 2026-11-06 09:51:45 +09:00
1 parent 543814cbd6
commit 89de2f7ea0
7 files changed
+165 -5

No files matched your search

@@ -24,6 +24,10 @@ Webの新規作成・クイック追加・サブタスク追加でも既定値
### メール本文のWiki保存
v0.9.0は任意の `wiki_suffix` をcaptureに追加する。CLI v0.4.0はML連番なら `リスト名/連番`、それ以外は `YYYY/MM/UUID`(Message-IDあり)または `YYYY/MM/原文SHA-256`(なし)を送る。Dateの記載タイムゾーンの年月を採用し、不正・欠落時のみ初回取り込みUTC年月を用いる。判定はCLIで確定してoutboxに保持する。リスト名はASCII英数字で始まる100文字以内の英数字・`._-`、連番は先頭ゼロ除去後最大20桁。複数表記は先頭を採用する。不正な表記は年月方式へフォールバックする。
サーバーはMLの2階層または年月の3階層だけを受理し、絶対Path・`.`・`..`・予約セグメント・query/fragment・不正年月・不正識別子を拒否する。解決した親Pathの下へ結合し、その完全Pathのread/create/write権限を検証する。衝突は409で、別Message-IDのタスクも部分作成しない。wiki_suffixはwiki_body指定時のみ許可する。省略時は従来の `mail-request_id` とし、旧receiptのhash・保存先を維持する。新CLIは旧サーバーに暗黙フォールバックしない。既存ページの自動移動、同じ件名・同じ連番の上書きは行わない。設定変更や月替わりで保留要求を書き換えず、異なるオプションでの上書きを拒否する。
v0.8.0ではcaptureに省略可能な `wiki_body` と `wiki_prefix` を追加する。wiki_bodyを省略した従来クライアントの契約・receipt hashは変えない。本文を指定した場合、親Pathは要求のwiki_prefix、個人preferencesのwiki_prefix、選択プロジェクトの関連WikiとNotesSubpath(未設定はtasks)の順に解決する。いずれもなければ422 `wiki_prefix_required`。空の明示指定も拒否する。個人設定はDomain・本人単位で、既存preferences JSONに保存するため新規schema migrationは不要。
本文は256 KiBまでのUTF-8テキスト。H1に件名、preに本文をHTMLエスケープして格納し、波括弧も実体参照にしてマクロを実行しない。v0.8.1以降のPathは `親Path/mail-request_id`。CLIは接続先・本人・Message-ID(なければ原文SHA-256)からrequest_idを生成するため、Path末尾は件名・日本語の文字数・ML連番に依存しないASCIIになる。Message-ID自体はURLへ露出しない。既存Pathとの競合は409で、既存ページへ追記・上書きしない。新規Wikiのobject/path/revisionとactivity outbox、タスクのPageUUID、capture receiptは同一DBトランザクションで保存する。rollback時に未参照のcontent-addressed Documentファイルが残る可能性は既存DocumentStoreと同様だが、公開可能なページや片方だけのタスクは残さない。
+74
View File
@@ -0,0 +1,74 @@
package server
import (
"context"
"database/sql"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/google/uuid"
"pageswiki/internal/store"
)
func TestTaskMailLayouts(t *testing.T) {
s, token, domain, actor := taskFixture(t)
for _, suffix := range []string{"hogehoge/123", "v/0", "2026/01/" + uuid.NewString(), "2026/12/" + strings.Repeat("a", 64)} {
id := uuid.NewString()
input := map[string]any{"request_id": id, "title": "日本語の件名", "wiki_body": "Body", "wiki_prefix": "/mail", "wiki_suffix": suffix}
w := taskRequest(s, token, "POST", "capture", input, "")
if w.Code != 201 {
t.Fatal(suffix, w.Code, w.Body)
}
var response struct {
Path string `json:"page_path"`
Item store.TaskRecord `json:"item"`
}
if e := json.Unmarshal(w.Body.Bytes(), &response); e != nil {
t.Fatal(e)
}
if response.Path != "/mail/"+suffix {
t.Fatal(response.Path)
}
if w = taskRequest(s, token, "POST", "capture", input, ""); w.Code != 201 {
t.Fatal("replay", w.Code, w.Body)
}
// A different Message-ID produces a different request UUID, never an overwrite.
other := uuid.NewString()
input["request_id"] = other
input["wiki_body"] = "Different mail"
if w = taskRequest(s, token, "POST", "capture", input, ""); w.Code != 409 {
t.Fatal("collision", w.Code, w.Body)
}
if _, e := s.store.GetTask(context.Background(), domain, actor, other); !errors.Is(e, sql.ErrNoRows) {
t.Fatal("partial task", e)
}
rev, e := s.store.GetLatestRevision(response.Item.PageUUID)
if e != nil || strings.Contains(rev.Body, "Different") {
t.Fatal("page overwritten", e)
}
}
for _, suffix := range []string{"", "../123", "/list/123", "a/../../123", "list/001", "list/-1", "list/1?x", "list%2f/1", "list/1/2", "_/123", "2026/13/" + uuid.NewString(), "0000/01/" + uuid.NewString(), "2026/01/not-an-id"} {
input := map[string]any{"request_id": uuid.NewString(), "title": "Bad", "wiki_body": "Body", "wiki_prefix": "/mail", "wiki_suffix": suffix}
if w := taskRequest(s, token, "POST", "capture", input, ""); w.Code != 422 {
t.Fatal(suffix, w.Code, w.Body)
}
}
input := map[string]any{"request_id": uuid.NewString(), "title": "No body", "wiki_suffix": "list/123"}
if w := taskRequest(s, token, "POST", "capture", input, ""); w.Code != 422 {
t.Fatal(w.Code, w.Body)
}
other, e := s.store.CreatePasswordAccount(domain, "mail-layout-other", "fixture password long", "Other", "")
if e != nil {
t.Fatal(e)
}
otherToken, e := s.store.CreateAuthSession(domain, other.UUID)
if e != nil {
t.Fatal(e)
}
input = map[string]any{"request_id": uuid.NewString(), "title": "No Wiki permission", "wiki_body": "Body", "wiki_prefix": "/mail", "wiki_suffix": "private/123"}
if w := taskRequest(s, otherToken, "POST", "capture", input, ""); w.Code != 403 {
t.Fatal(w.Code, w.Body)
}
}
+9 -1
View File
@@ -69,6 +69,7 @@ type taskCaptureInput struct {
Complexity *int `json:"complexity"`
WikiBody *string `json:"wiki_body,omitempty"`
WikiPrefix *string `json:"wiki_prefix,omitempty"`
WikiSuffix *string `json:"wiki_suffix,omitempty"`
}
func (s *Server) handleTaskProductivity(w http.ResponseWriter, r *http.Request, actor string, active bool, kind string) {
@@ -260,6 +261,13 @@ func (s *Server) handleTaskProductivity(w http.ResponseWriter, r *http.Request,
return
}
target = store.TaskCaptureWikiPath(prefix, item.Title, input.RequestID)
if input.WikiSuffix != nil {
if !store.ValidTaskMailSuffix(*input.WikiSuffix) {
taskError(w, store.ErrTaskInvalid)
return
}
target = strings.TrimRight(prefix, "/") + "/" + *input.WikiSuffix
}
for _, capability := range []string{"read", "create", "write"} {
if s.authorizePathRequest(r, capability, target) != nil {
taskError(w, store.ErrAccessDenied)
@@ -267,7 +275,7 @@ func (s *Server) handleTaskProductivity(w http.ResponseWriter, r *http.Request,
}
}
body = *input.WikiBody
} else if input.WikiPrefix != nil {
} else if input.WikiPrefix != nil || input.WikiSuffix != nil {
taskError(w, store.ErrTaskInvalid)
return
}
+9
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"html"
"path"
"regexp"
"strings"
"time"
"unicode"
@@ -40,6 +41,14 @@ func TaskCaptureWikiPath(prefix, title, request string) string {
return strings.TrimRight(prefix, "/") + "/mail-" + request
}
var mailListSuffix = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,99}/(0|[1-9][0-9]{0,19})$`)
var mailMonthSuffix = regexp.MustCompile(`^[0-9]{4}/(0[1-9]|1[0-2])/([0-9a-f]{64}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$`)
// Only the two mail layouts are accepted; no arbitrary relative path traversal.
func ValidTaskMailSuffix(s string) bool {
return mailListSuffix.MatchString(s) || (mailMonthSuffix.MatchString(s) && !strings.HasPrefix(s, "0000/"))
}
func (s *Store) createCapturedTaskWiki(ctx context.Context, tx *sql.Tx, domain, actor string, item *TaskRecord, target, body string) error {
if target == "" || !ValidTaskWikiPrefix(path.Dir(target)) || path.Clean(target) != target || item.PageUUID != "" || len(body) > 256<<10 || !utf8.ValidString(body) || strings.IndexByte(body, 0) >= 0 {
return ErrTaskInvalid
+27 -3
View File
@@ -3,6 +3,8 @@
import argparse
import email.policy
import email.parser
import email.utils
import datetime
import uuid
import base64
import contextlib
@@ -30,7 +32,7 @@ import urllib.parse as url
import urllib.request as http
import webbrowser
VERSION = '0.3.1'
VERSION = '0.4.0'
class Failure(Exception):
def __init__(self, message, status=0, payload=None):
@@ -506,6 +508,19 @@ def mail_subject(value):
value = match.group('reply') + value[match.end():]
def task_mail_suffix(raw, request_id, now=None):
message = email.parser.BytesParser(policy=email.policy.default).parsebytes(raw)
subject = ' '.join(safe_text(str(message.get('Subject') or '')).split())
marker = re.match(r'^(?:(?:re|fw|fwd)\s*:\s*)*\[([A-Za-z0-9][A-Za-z0-9._-]{0,99})[\s:#]+([0-9]{1,20})\]',subject,re.IGNORECASE)
if marker:
return marker[1]+'/'+str(int(marker[2]))
try: date = email.utils.parsedate_to_datetime(str(message.get('Date') or ''))
except (ValueError, TypeError, OverflowError): date = now or datetime.datetime.now(datetime.timezone.utc)
# Preserve the month in the sender's Date timezone, not the receiving host's.
ident = request_id if ' '.join(safe_text(str(message.get('Message-ID') or '')).split()) else hashlib.sha256(raw).hexdigest()
return f'{date.year:04d}/{date.month:02d}/{ident}'
def task_mail(raw, include_body=True, override_title=None):
if len(raw) > 2 << 20: raise Failure('Mail input exceeds 2 MiB; pipe one message at a time.')
message = email.parser.BytesParser(policy=email.policy.default).parsebytes(raw)
@@ -563,6 +578,7 @@ def tasks(client, profile, words):
def capture(payload):
pending=profile.data.setdefault('task_outbox',{})
if len(pending)>=20 and payload['request_id'] not in pending:raise Failure('20 pending captures retained. Retry them before adding more.')
if payload['request_id'] in pending and pending[payload['request_id']] != payload:raise Failure('A different draft with this request ID is retained. Use pages task retry '+payload['request_id'])
pending[payload['request_id']]=payload;profile.save()
try: result=client.json(TASK_API+'capture',method='POST',data=payload)
except Failure as e:
@@ -595,13 +611,21 @@ def tasks(client, profile, words):
payload={'request_id':getattr(a,'request_id',None) or str(uuid.uuid4()),'title':a.title,'notes':a.notes or ''}
if action=='from-mail':
if a.metadata_only and a.wiki_prefix is not None: raise Failure('--wiki-prefix cannot be combined with --metadata-only.')
title,notes,message_id,wiki_body=task_mail(sys.stdin.buffer.read((2<<20)+1),not a.metadata_only,a.title)
raw=sys.stdin.buffer.read((2<<20)+1)
title,notes,message_id,wiki_body=task_mail(raw,not a.metadata_only,a.title)
payload.update(title=a.title or title,notes=a.notes if a.notes is not None else notes,request_id=str(uuid.uuid5(uuid.NAMESPACE_URL,client.base+'\n'+profile.data.get('username','')+'\n'+message_id)))
if wiki_body is not None: payload['wiki_body']=wiki_body
if wiki_body is not None:
payload['wiki_body']=wiki_body
payload['wiki_suffix']=task_mail_suffix(raw,payload['request_id'])
if a.wiki_prefix is not None: payload['wiki_prefix']=a.wiki_prefix
for attr,key in [('project','project_uuid'),('assignee','assignee_uuid'),('due','due_on'),('start_on','start_on'),('planned','planned_on'),('review','review_on'),('estimate','estimate_minutes'),('complexity','complexity')]:
value=getattr(a,attr)
if value is not None:payload[key]=task_project(client,value) if attr=='project' else value
# A retry of the same pending mail must not drift across a month boundary
# when Date was missing. Other changed options still fail closed.
old=profile.data.get('task_outbox',{}).get(payload['request_id'])
if action=='from-mail' and old and {k:v for k,v in old.items() if k!='wiki_suffix'}=={k:v for k,v in payload.items() if k!='wiki_suffix'}:
payload=old.copy()
emit(capture(payload));return
if action=='project-add':
emit(client.json(TASK_API+'items',method='POST',data={'uuid':str(uuid.uuid4()),'kind':'project','title':a.title,'notes':a.notes,'status':'todo','bucket':'inbox','priority':1}));return
+33
View File
@@ -9,6 +9,39 @@ import unittest
m=runpy.run_path(str(Path(__file__).with_name('pages')))
class ProductivityTests(unittest.TestCase):
def test_mail_path_priority_and_dates(self):
import datetime,hashlib
ident='06cb3a24-61f8-4a9f-a0bf-89eea9632f25'
suffix=m['task_mail_suffix'];now=datetime.datetime(2030,2,1,tzinfo=datetime.timezone.utc)
for marker in ('[hogehoge 00123]','Re: [hogehoge:123]','Fwd: [hogehoge#123]'):
raw=('Subject: '+marker+' Title\r\n\r\nbody').encode()
self.assertEqual(suffix(raw,ident,now),'hogehoge/123')
raw=b'Subject: Plain\r\nDate: Sat, 31 Jan 2026 23:30:00 -0800\r\nMessage-ID: <id@example.invalid>\r\n\r\nbody'
self.assertEqual(suffix(raw,ident,now),'2026/01/'+ident)
noid=raw.replace(b'Message-ID: <id@example.invalid>\r\n',b'')
self.assertEqual(suffix(noid,ident,now),'2026/01/'+hashlib.sha256(noid).hexdigest())
for date in (b'',b'Date: broken\r\n'):
raw=b'Subject: [../unsafe 123] Title\r\n'+date+b'\r\nbody'
self.assertEqual(suffix(raw,ident,now),'2030/02/'+hashlib.sha256(raw).hexdigest())
def test_pending_mail_month_and_options_are_preserved(self):
from unittest.mock import patch
raw=b'Subject: Test\r\nMessage-ID: <same@example.invalid>\r\n\r\nbody'
calls=[]
def send(route,**kw):
calls.append(kw['data'].copy())
raise m['Failure']('offline')
client=SimpleNamespace(base='https://example.invalid',json=send)
profile=SimpleNamespace(data={'username':'test'},save=lambda:None)
for month in ('2030/01','2030/02'):
with patch.dict(m['tasks'].__globals__,task_mail_suffix=lambda raw,ident:month+'/'+ident),patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw))),contextlib.redirect_stderr(io.StringIO()),self.assertRaises(m['Failure']):
m['tasks'](client,profile,['from-mail'])
self.assertEqual(calls[0],calls[1])
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw))),self.assertRaises(m['Failure']):
m['tasks'](client,profile,['from-mail','--wiki-prefix','/changed'])
self.assertEqual(len(calls),2)
self.assertEqual(next(iter(profile.data['task_outbox'].values())),calls[0])
def test_mail_privacy_and_long_subject_override(self):
raw=b'Subject: Test\r\nFrom: Person <person@example.invalid>\r\nMessage-ID: <id@example.invalid>\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPRIVATE BODY'
title,notes,ident,body=m['task_mail'](raw)
+9 -1
View File
@@ -34,7 +34,7 @@ def run():
mail=message.as_bytes()
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))):
captured=task('from-mail')
assert captured['item']['page_uuid'] and captured['page_path']=='/private/mail/mail-'+captured['item']['uuid']
assert captured['item']['page_uuid'] and captured['page_path']=='/private/mail/hogehoge/123'
assert captured['item']['title']=='日本語の件名と依頼'
assert 'Private mail body' not in captured['item']['notes']
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))):
@@ -43,6 +43,14 @@ def run():
assert note['linked'] and note['path']==captured['page_path']
raw,_=client.raw('path:'+captured['page_path'])
assert '日本語の件名と依頼' in raw.decode() and 'Private mail body' in raw.decode()
message.replace_header('Subject','連番なしの日本語');message.replace_header('Message-ID','<other@example.invalid>');message['Date']='Sat, 31 Jan 2026 23:30:00 -0800'
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(message.as_bytes()))):dated=task('from-mail')
assert dated['page_path']=='/private/mail/2026/01/'+dated['item']['uuid']
del message['Message-ID']
import hashlib
raw_mail=message.as_bytes()
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw_mail))):hashed=task('from-mail')
assert hashed['page_path']=='/private/mail/2026/01/'+hashlib.sha256(raw_mail).hexdigest()
path=Path(fixture['root'])/'person.vcf'
path.write_text('BEGIN:VCARD\r\nVERSION:4.0\r\nUID:urn:test:cli\r\nFN:CLI Person\r\nEMAIL:cli@example.invalid\r\nEND:VCARD\r\n')
output=io.StringIO()