From 1a451023f09f483f5efff450dc2c951c925eb999 Mon Sep 17 00:00:00 2001 From: Anya Tokugawa Date: Fri, 6 Nov 2026 09:51:45 +0900 Subject: [PATCH] feat: support mailing-list and monthly mail capture paths --- docs/design/ja/task-productivity-contacts.md | 4 ++ internal/server/task_mail_layout_test.go | 74 ++++++++++++++++++++ internal/server/task_productivity.go | 10 ++- internal/store/task_capture_wiki.go | 9 +++ pages-cli/pages | 30 +++++++- pages-cli/test_productivity.py | 33 +++++++++ pages-cli/test_task_integration.py | 10 ++- 7 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 internal/server/task_mail_layout_test.go diff --git a/docs/design/ja/task-productivity-contacts.md b/docs/design/ja/task-productivity-contacts.md index e56003e..54bcd65 100644 --- a/docs/design/ja/task-productivity-contacts.md +++ b/docs/design/ja/task-productivity-contacts.md @@ -24,6 +24,10 @@ Webの新規作成・クイック追加・サブタスク追加でも既定値 ### メール本文のWiki保存 +v0.9.0は任意の `wiki_suffix` をcaptureに追加する。CLI v0.4.0はML連番なら `リスト名/連番`、それ以外は `YYYY/MM/UUID`(Message-IDあり)または `YYYY/MM/原文SHA-256`(なし)を送る。Dateの記載タイムゾーンの年月を採用し、不正・欠落時のみ初回取り込みUTC年月を用いる。判定はCLIで確定してoutboxに保持する。リスト名はASCII英数字で始まる100文字以内の英数字・`._-`、連番は先頭ゼロ除去後最大20桁。複数表記は先頭を採用する。不正な表記は年月方式へフォールバックする。 + +サーバーはMLの2階層または年月の3階層だけを受理し、絶対Path・`.`・`..`・予約セグメント・query/fragment・不正年月・不正識別子を拒否する。解決した親Pathの下へ結合し、その完全Pathのread/create/write権限を検証する。衝突は409で、別Message-IDのタスクも部分作成しない。wiki_suffixはwiki_body指定時のみ許可する。省略時は従来の `mail-request_id` とし、旧receiptのhash・保存先を維持する。新CLIは旧サーバーに暗黙フォールバックしない。既存ページの自動移動、同じ件名・同じ連番の上書きは行わない。設定変更や月替わりで保留要求を書き換えず、異なるオプションでの上書きを拒否する。 + v0.8.0ではcaptureに省略可能な `wiki_body` と `wiki_prefix` を追加する。wiki_bodyを省略した従来クライアントの契約・receipt hashは変えない。本文を指定した場合、親Pathは要求のwiki_prefix、個人preferencesのwiki_prefix、選択プロジェクトの関連WikiとNotesSubpath(未設定はtasks)の順に解決する。いずれもなければ422 `wiki_prefix_required`。空の明示指定も拒否する。個人設定はDomain・本人単位で、既存preferences JSONに保存するため新規schema migrationは不要。 本文は256 KiBまでのUTF-8テキスト。H1に件名、preに本文をHTMLエスケープして格納し、波括弧も実体参照にしてマクロを実行しない。v0.8.1以降のPathは `親Path/mail-request_id`。CLIは接続先・本人・Message-ID(なければ原文SHA-256)からrequest_idを生成するため、Path末尾は件名・日本語の文字数・ML連番に依存しないASCIIになる。Message-ID自体はURLへ露出しない。既存Pathとの競合は409で、既存ページへ追記・上書きしない。新規Wikiのobject/path/revisionとactivity outbox、タスクのPageUUID、capture receiptは同一DBトランザクションで保存する。rollback時に未参照のcontent-addressed Documentファイルが残る可能性は既存DocumentStoreと同様だが、公開可能なページや片方だけのタスクは残さない。 diff --git a/internal/server/task_mail_layout_test.go b/internal/server/task_mail_layout_test.go new file mode 100644 index 0000000..9e64d50 --- /dev/null +++ b/internal/server/task_mail_layout_test.go @@ -0,0 +1,74 @@ +package server + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/google/uuid" + "pageswiki/internal/store" +) + +func TestTaskMailLayouts(t *testing.T) { + s, token, domain, actor := taskFixture(t) + for _, suffix := range []string{"hogehoge/123", "v/0", "2026/01/" + uuid.NewString(), "2026/12/" + strings.Repeat("a", 64)} { + id := uuid.NewString() + input := map[string]any{"request_id": id, "title": "日本語の件名", "wiki_body": "Body", "wiki_prefix": "/mail", "wiki_suffix": suffix} + w := taskRequest(s, token, "POST", "capture", input, "") + if w.Code != 201 { + t.Fatal(suffix, w.Code, w.Body) + } + var response struct { + Path string `json:"page_path"` + Item store.TaskRecord `json:"item"` + } + if e := json.Unmarshal(w.Body.Bytes(), &response); e != nil { + t.Fatal(e) + } + if response.Path != "/mail/"+suffix { + t.Fatal(response.Path) + } + if w = taskRequest(s, token, "POST", "capture", input, ""); w.Code != 201 { + t.Fatal("replay", w.Code, w.Body) + } + // A different Message-ID produces a different request UUID, never an overwrite. + other := uuid.NewString() + input["request_id"] = other + input["wiki_body"] = "Different mail" + if w = taskRequest(s, token, "POST", "capture", input, ""); w.Code != 409 { + t.Fatal("collision", w.Code, w.Body) + } + if _, e := s.store.GetTask(context.Background(), domain, actor, other); !errors.Is(e, sql.ErrNoRows) { + t.Fatal("partial task", e) + } + rev, e := s.store.GetLatestRevision(response.Item.PageUUID) + if e != nil || strings.Contains(rev.Body, "Different") { + t.Fatal("page overwritten", e) + } + } + for _, suffix := range []string{"", "../123", "/list/123", "a/../../123", "list/001", "list/-1", "list/1?x", "list%2f/1", "list/1/2", "_/123", "2026/13/" + uuid.NewString(), "0000/01/" + uuid.NewString(), "2026/01/not-an-id"} { + input := map[string]any{"request_id": uuid.NewString(), "title": "Bad", "wiki_body": "Body", "wiki_prefix": "/mail", "wiki_suffix": suffix} + if w := taskRequest(s, token, "POST", "capture", input, ""); w.Code != 422 { + t.Fatal(suffix, w.Code, w.Body) + } + } + input := map[string]any{"request_id": uuid.NewString(), "title": "No body", "wiki_suffix": "list/123"} + if w := taskRequest(s, token, "POST", "capture", input, ""); w.Code != 422 { + t.Fatal(w.Code, w.Body) + } + other, e := s.store.CreatePasswordAccount(domain, "mail-layout-other", "fixture password long", "Other", "") + if e != nil { + t.Fatal(e) + } + otherToken, e := s.store.CreateAuthSession(domain, other.UUID) + if e != nil { + t.Fatal(e) + } + input = map[string]any{"request_id": uuid.NewString(), "title": "No Wiki permission", "wiki_body": "Body", "wiki_prefix": "/mail", "wiki_suffix": "private/123"} + if w := taskRequest(s, otherToken, "POST", "capture", input, ""); w.Code != 403 { + t.Fatal(w.Code, w.Body) + } +} diff --git a/internal/server/task_productivity.go b/internal/server/task_productivity.go index 3bb8ff5..e803c7a 100644 --- a/internal/server/task_productivity.go +++ b/internal/server/task_productivity.go @@ -69,6 +69,7 @@ type taskCaptureInput struct { Complexity *int `json:"complexity"` WikiBody *string `json:"wiki_body,omitempty"` WikiPrefix *string `json:"wiki_prefix,omitempty"` + WikiSuffix *string `json:"wiki_suffix,omitempty"` } func (s *Server) handleTaskProductivity(w http.ResponseWriter, r *http.Request, actor string, active bool, kind string) { @@ -260,6 +261,13 @@ func (s *Server) handleTaskProductivity(w http.ResponseWriter, r *http.Request, return } target = store.TaskCaptureWikiPath(prefix, item.Title, input.RequestID) + if input.WikiSuffix != nil { + if !store.ValidTaskMailSuffix(*input.WikiSuffix) { + taskError(w, store.ErrTaskInvalid) + return + } + target = strings.TrimRight(prefix, "/") + "/" + *input.WikiSuffix + } for _, capability := range []string{"read", "create", "write"} { if s.authorizePathRequest(r, capability, target) != nil { taskError(w, store.ErrAccessDenied) @@ -267,7 +275,7 @@ func (s *Server) handleTaskProductivity(w http.ResponseWriter, r *http.Request, } } body = *input.WikiBody - } else if input.WikiPrefix != nil { + } else if input.WikiPrefix != nil || input.WikiSuffix != nil { taskError(w, store.ErrTaskInvalid) return } diff --git a/internal/store/task_capture_wiki.go b/internal/store/task_capture_wiki.go index ff84aa6..92ac3cd 100644 --- a/internal/store/task_capture_wiki.go +++ b/internal/store/task_capture_wiki.go @@ -6,6 +6,7 @@ import ( "encoding/json" "html" "path" + "regexp" "strings" "time" "unicode" @@ -40,6 +41,14 @@ func TaskCaptureWikiPath(prefix, title, request string) string { return strings.TrimRight(prefix, "/") + "/mail-" + request } +var mailListSuffix = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,99}/(0|[1-9][0-9]{0,19})$`) +var mailMonthSuffix = regexp.MustCompile(`^[0-9]{4}/(0[1-9]|1[0-2])/([0-9a-f]{64}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$`) + +// Only the two mail layouts are accepted; no arbitrary relative path traversal. +func ValidTaskMailSuffix(s string) bool { + return mailListSuffix.MatchString(s) || (mailMonthSuffix.MatchString(s) && !strings.HasPrefix(s, "0000/")) +} + func (s *Store) createCapturedTaskWiki(ctx context.Context, tx *sql.Tx, domain, actor string, item *TaskRecord, target, body string) error { if target == "" || !ValidTaskWikiPrefix(path.Dir(target)) || path.Clean(target) != target || item.PageUUID != "" || len(body) > 256<<10 || !utf8.ValidString(body) || strings.IndexByte(body, 0) >= 0 { return ErrTaskInvalid diff --git a/pages-cli/pages b/pages-cli/pages index bf6857c..2e45c71 100755 --- a/pages-cli/pages +++ b/pages-cli/pages @@ -3,6 +3,8 @@ import argparse import email.policy import email.parser +import email.utils +import datetime import uuid import base64 import contextlib @@ -30,7 +32,7 @@ import urllib.parse as url import urllib.request as http import webbrowser -VERSION = '0.3.1' +VERSION = '0.4.0' class Failure(Exception): def __init__(self, message, status=0, payload=None): @@ -506,6 +508,19 @@ def mail_subject(value): value = match.group('reply') + value[match.end():] +def task_mail_suffix(raw, request_id, now=None): + message = email.parser.BytesParser(policy=email.policy.default).parsebytes(raw) + subject = ' '.join(safe_text(str(message.get('Subject') or '')).split()) + marker = re.match(r'^(?:(?:re|fw|fwd)\s*:\s*)*\[([A-Za-z0-9][A-Za-z0-9._-]{0,99})[\s:#]+([0-9]{1,20})\]',subject,re.IGNORECASE) + if marker: + return marker[1]+'/'+str(int(marker[2])) + try: date = email.utils.parsedate_to_datetime(str(message.get('Date') or '')) + except (ValueError, TypeError, OverflowError): date = now or datetime.datetime.now(datetime.timezone.utc) + # Preserve the month in the sender's Date timezone, not the receiving host's. + ident = request_id if ' '.join(safe_text(str(message.get('Message-ID') or '')).split()) else hashlib.sha256(raw).hexdigest() + return f'{date.year:04d}/{date.month:02d}/{ident}' + + def task_mail(raw, include_body=True, override_title=None): if len(raw) > 2 << 20: raise Failure('Mail input exceeds 2 MiB; pipe one message at a time.') message = email.parser.BytesParser(policy=email.policy.default).parsebytes(raw) @@ -563,6 +578,7 @@ def tasks(client, profile, words): def capture(payload): pending=profile.data.setdefault('task_outbox',{}) if len(pending)>=20 and payload['request_id'] not in pending:raise Failure('20 pending captures retained. Retry them before adding more.') + if payload['request_id'] in pending and pending[payload['request_id']] != payload:raise Failure('A different draft with this request ID is retained. Use pages task retry '+payload['request_id']) pending[payload['request_id']]=payload;profile.save() try: result=client.json(TASK_API+'capture',method='POST',data=payload) except Failure as e: @@ -595,13 +611,21 @@ def tasks(client, profile, words): payload={'request_id':getattr(a,'request_id',None) or str(uuid.uuid4()),'title':a.title,'notes':a.notes or ''} if action=='from-mail': if a.metadata_only and a.wiki_prefix is not None: raise Failure('--wiki-prefix cannot be combined with --metadata-only.') - title,notes,message_id,wiki_body=task_mail(sys.stdin.buffer.read((2<<20)+1),not a.metadata_only,a.title) + raw=sys.stdin.buffer.read((2<<20)+1) + title,notes,message_id,wiki_body=task_mail(raw,not a.metadata_only,a.title) payload.update(title=a.title or title,notes=a.notes if a.notes is not None else notes,request_id=str(uuid.uuid5(uuid.NAMESPACE_URL,client.base+'\n'+profile.data.get('username','')+'\n'+message_id))) - if wiki_body is not None: payload['wiki_body']=wiki_body + if wiki_body is not None: + payload['wiki_body']=wiki_body + payload['wiki_suffix']=task_mail_suffix(raw,payload['request_id']) if a.wiki_prefix is not None: payload['wiki_prefix']=a.wiki_prefix for attr,key in [('project','project_uuid'),('assignee','assignee_uuid'),('due','due_on'),('start_on','start_on'),('planned','planned_on'),('review','review_on'),('estimate','estimate_minutes'),('complexity','complexity')]: value=getattr(a,attr) if value is not None:payload[key]=task_project(client,value) if attr=='project' else value + # A retry of the same pending mail must not drift across a month boundary + # when Date was missing. Other changed options still fail closed. + old=profile.data.get('task_outbox',{}).get(payload['request_id']) + if action=='from-mail' and old and {k:v for k,v in old.items() if k!='wiki_suffix'}=={k:v for k,v in payload.items() if k!='wiki_suffix'}: + payload=old.copy() emit(capture(payload));return if action=='project-add': emit(client.json(TASK_API+'items',method='POST',data={'uuid':str(uuid.uuid4()),'kind':'project','title':a.title,'notes':a.notes,'status':'todo','bucket':'inbox','priority':1}));return diff --git a/pages-cli/test_productivity.py b/pages-cli/test_productivity.py index 26f8ed4..0d00079 100644 --- a/pages-cli/test_productivity.py +++ b/pages-cli/test_productivity.py @@ -9,6 +9,39 @@ import unittest m=runpy.run_path(str(Path(__file__).with_name('pages'))) class ProductivityTests(unittest.TestCase): + def test_mail_path_priority_and_dates(self): + import datetime,hashlib + ident='06cb3a24-61f8-4a9f-a0bf-89eea9632f25' + suffix=m['task_mail_suffix'];now=datetime.datetime(2030,2,1,tzinfo=datetime.timezone.utc) + for marker in ('[hogehoge 00123]','Re: [hogehoge:123]','Fwd: [hogehoge#123]'): + raw=('Subject: '+marker+' Title\r\n\r\nbody').encode() + self.assertEqual(suffix(raw,ident,now),'hogehoge/123') + raw=b'Subject: Plain\r\nDate: Sat, 31 Jan 2026 23:30:00 -0800\r\nMessage-ID: \r\n\r\nbody' + self.assertEqual(suffix(raw,ident,now),'2026/01/'+ident) + noid=raw.replace(b'Message-ID: \r\n',b'') + self.assertEqual(suffix(noid,ident,now),'2026/01/'+hashlib.sha256(noid).hexdigest()) + for date in (b'',b'Date: broken\r\n'): + raw=b'Subject: [../unsafe 123] Title\r\n'+date+b'\r\nbody' + self.assertEqual(suffix(raw,ident,now),'2030/02/'+hashlib.sha256(raw).hexdigest()) + + def test_pending_mail_month_and_options_are_preserved(self): + from unittest.mock import patch + raw=b'Subject: Test\r\nMessage-ID: \r\n\r\nbody' + calls=[] + def send(route,**kw): + calls.append(kw['data'].copy()) + raise m['Failure']('offline') + client=SimpleNamespace(base='https://example.invalid',json=send) + profile=SimpleNamespace(data={'username':'test'},save=lambda:None) + for month in ('2030/01','2030/02'): + with patch.dict(m['tasks'].__globals__,task_mail_suffix=lambda raw,ident:month+'/'+ident),patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw))),contextlib.redirect_stderr(io.StringIO()),self.assertRaises(m['Failure']): + m['tasks'](client,profile,['from-mail']) + self.assertEqual(calls[0],calls[1]) + with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw))),self.assertRaises(m['Failure']): + m['tasks'](client,profile,['from-mail','--wiki-prefix','/changed']) + self.assertEqual(len(calls),2) + self.assertEqual(next(iter(profile.data['task_outbox'].values())),calls[0]) + def test_mail_privacy_and_long_subject_override(self): raw=b'Subject: Test\r\nFrom: Person \r\nMessage-ID: \r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPRIVATE BODY' title,notes,ident,body=m['task_mail'](raw) diff --git a/pages-cli/test_task_integration.py b/pages-cli/test_task_integration.py index d9ef35d..21fe074 100644 --- a/pages-cli/test_task_integration.py +++ b/pages-cli/test_task_integration.py @@ -34,7 +34,7 @@ def run(): mail=message.as_bytes() with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))): captured=task('from-mail') - assert captured['item']['page_uuid'] and captured['page_path']=='/private/mail/mail-'+captured['item']['uuid'] + assert captured['item']['page_uuid'] and captured['page_path']=='/private/mail/hogehoge/123' assert captured['item']['title']=='日本語の件名と依頼' assert 'Private mail body' not in captured['item']['notes'] with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))): @@ -43,6 +43,14 @@ def run(): assert note['linked'] and note['path']==captured['page_path'] raw,_=client.raw('path:'+captured['page_path']) assert '日本語の件名と依頼' in raw.decode() and 'Private mail body' in raw.decode() + message.replace_header('Subject','連番なしの日本語');message.replace_header('Message-ID','');message['Date']='Sat, 31 Jan 2026 23:30:00 -0800' + with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(message.as_bytes()))):dated=task('from-mail') + assert dated['page_path']=='/private/mail/2026/01/'+dated['item']['uuid'] + del message['Message-ID'] + import hashlib + raw_mail=message.as_bytes() + with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw_mail))):hashed=task('from-mail') + assert hashed['page_path']=='/private/mail/2026/01/'+hashlib.sha256(raw_mail).hexdigest() path=Path(fixture['root'])/'person.vcf' path.write_text('BEGIN:VCARD\r\nVERSION:4.0\r\nUID:urn:test:cli\r\nFN:CLI Person\r\nEMAIL:cli@example.invalid\r\nEND:VCARD\r\n') output=io.StringIO()