diff --git a/docs/design/ja/task-productivity-contacts.md b/docs/design/ja/task-productivity-contacts.md index a771c88..e56003e 100644 --- a/docs/design/ja/task-productivity-contacts.md +++ b/docs/design/ja/task-productivity-contacts.md @@ -26,7 +26,9 @@ Webの新規作成・クイック追加・サブタスク追加でも既定値 v0.8.0ではcaptureに省略可能な `wiki_body` と `wiki_prefix` を追加する。wiki_bodyを省略した従来クライアントの契約・receipt hashは変えない。本文を指定した場合、親Pathは要求のwiki_prefix、個人preferencesのwiki_prefix、選択プロジェクトの関連WikiとNotesSubpath(未設定はtasks)の順に解決する。いずれもなければ422 `wiki_prefix_required`。空の明示指定も拒否する。個人設定はDomain・本人単位で、既存preferences JSONに保存するため新規schema migrationは不要。 -本文は256 KiBまでのUTF-8テキスト。H1に件名、preに本文をHTMLエスケープして格納し、波括弧も実体参照にしてマクロを実行しない。Pathは `親Path/安全化した件名--request_id`。既存Pathとの競合は409で、既存ページへ追記・上書きしない。新規Wikiのobject/path/revisionとactivity outbox、タスクのPageUUID、capture receiptは同一DBトランザクションで保存する。rollback時に未参照のcontent-addressed Documentファイルが残る可能性は既存DocumentStoreと同様だが、公開可能なページや片方だけのタスクは残さない。 +本文は256 KiBまでのUTF-8テキスト。H1に件名、preに本文をHTMLエスケープして格納し、波括弧も実体参照にしてマクロを実行しない。v0.8.1以降のPathは `親Path/mail-request_id`。CLIは接続先・本人・Message-ID(なければ原文SHA-256)からrequest_idを生成するため、Path末尾は件名・日本語の文字数・ML連番に依存しないASCIIになる。Message-ID自体はURLへ露出しない。既存Pathとの競合は409で、既存ページへ追記・上書きしない。新規Wikiのobject/path/revisionとactivity outbox、タスクのPageUUID、capture receiptは同一DBトランザクションで保存する。rollback時に未参照のcontent-addressed Documentファイルが残る可能性は既存DocumentStoreと同様だが、公開可能なページや片方だけのタスクは残さない。 + +v0.8.0の件名入りPathは自動移動しない。既存receiptの同一要求再送は元のページを返す。CLI v0.3.1は先頭のML連番を除去し、長い自動件名は300 UTF-8 bytes以内に文字境界で省略、変更前のSubjectをWiki本文へ保持する。旧版で保存済みの要求を新しい件名・本文で再作成すると412となるため、通信失敗した要求は保存済みoutboxのretryを使う。サーバーAPIの入力フィールド・receipt hash形式は変更しない。 保存前に対象Pathへのread/create/write権限を検証する。個人タスクの所有権はWikiへの権限を付与しない。Wiki本文の公開範囲は保存先のACLに従うため、個人設定UIに注意を表示する。再送は既定値の再解決より前にreceiptを確認し、既定Pathの変更で重複作成しない。成功応答はitemとpage_pathを返し、タスクの既存メモ操作から関連Wikiへ移動できる。 diff --git a/internal/server/task_capture_wiki_test.go b/internal/server/task_capture_wiki_test.go index 6fd38ec..0b5f0c8 100644 --- a/internal/server/task_capture_wiki_test.go +++ b/internal/server/task_capture_wiki_test.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/json" "errors" + "net/http/httptest" + "net/url" "strings" "testing" "time" @@ -15,6 +17,7 @@ import ( func TestTaskMailWikiCapture(t *testing.T) { s, token, domain, actor := taskFixture(t) + s.mux.HandleFunc("/v/raw/", s.handleRawView) ctx := context.Background() p, err := s.store.TaskPreferences(ctx, domain, actor) if err != nil { @@ -51,6 +54,29 @@ func TestTaskMailWikiCapture(t *testing.T) { if err != nil || !strings.Contains(rev.Body, "<script>bad()</script>") || strings.Contains(rev.Body, "{{children}}") { t.Fatal(rev, err) } + // Existing Japanese paths are not renamed or broken by receipt replay. + legacy := "/private/mail/日本語の件名 [list 123]--" + id + if _, err = s.store.DB.Exec(`UPDATE paths SET path=? WHERE domain_uuid=? AND object_uuid=?`, legacy, domain, response.Item.PageUUID); err != nil { + t.Fatal(err) + } + note := taskRequest(s, token, "GET", "items/"+id+"/note", nil, "") + var links struct { + URL string `json:"url"` + } + if err = json.Unmarshal(note.Body.Bytes(), &links); err != nil { + t.Fatal(err) + } + u, err := url.Parse(links.URL) + if err != nil || u.Path != legacy || u.RawQuery != "" || u.Fragment != "" { + t.Fatal("Japanese note URL", links.URL, err) + } + request := httptest.NewRequest("GET", "http://localhost/v/raw/path:"+links.URL, nil) + request.Header.Set("Authorization", "Bearer "+token) + rawResult := httptest.NewRecorder() + s.withAuthorization(s.mux).ServeHTTP(rawResult, request) + if rawResult.Code != 200 || rawResult.Body.String() != rev.Body { + t.Fatal("Japanese path fetch", rawResult.Code) + } // A committed retry remains stable even if the preference changes. p, err = s.store.TaskPreferences(ctx, domain, actor) if err != nil { @@ -63,6 +89,9 @@ func TestTaskMailWikiCapture(t *testing.T) { if w = taskRequest(s, token, "POST", "capture", input, ""); w.Code != 201 || !strings.Contains(w.Body.String(), response.Item.PageUUID) { t.Fatal(w.Code, w.Body) } + if !strings.Contains(w.Body.String(), legacy) { + t.Fatal("retry changed existing path", w.Body) + } input["wiki_body"] = "different" if w = taskRequest(s, token, "POST", "capture", input, ""); w.Code != 412 { t.Fatal(w.Code, w.Body) @@ -74,6 +103,18 @@ func TestTaskMailWikiCapture(t *testing.T) { } } +func TestMailWikiPathDoesNotContainSubject(t *testing.T) { + id := uuid.NewString() + for _, title := range []string{"日本語", "[hogehoge 123] 件名", strings.Repeat("長い", 100), "?#/\\% []"} { + if got := store.TaskCaptureWikiPath("/mail", title, id); got != "/mail/mail-"+id { + t.Fatal(got) + } + } + if store.TaskCaptureWikiPath("/mail", "日本語", uuid.NewString()) == store.TaskCaptureWikiPath("/mail", "日本語", id) { + t.Fatal("different mails collided") + } +} + func TestTaskMailWikiRollbackAndAccess(t *testing.T) { s, token, domain, actor := taskFixture(t) ctx := context.Background() diff --git a/internal/server/task_cli_test.go b/internal/server/task_cli_test.go index 9410db1..3ab7ceb 100644 --- a/internal/server/task_cli_test.go +++ b/internal/server/task_cli_test.go @@ -16,6 +16,7 @@ func TestTaskCLIHTTPIntegration(t *testing.T) { t.Skip("python3 not installed") } s, token, _, _ := taskFixture(t) + s.mux.HandleFunc("/v/raw/", s.handleRawView) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { r.Host = "localhost" s.withAuthorization(s.mux).ServeHTTP(w, r) diff --git a/internal/store/task_capture_wiki.go b/internal/store/task_capture_wiki.go index 21380dd..ff84aa6 100644 --- a/internal/store/task_capture_wiki.go +++ b/internal/store/task_capture_wiki.go @@ -34,11 +34,10 @@ func ValidTaskWikiPrefix(p string) bool { return true } -// A request UUID suffix separates equal subjects and makes the target stable. +// The CLI derives the request UUID from origin, user and Message-ID (or a raw +// mail hash). Keep subjects in the title, never in the URL's identity segment. func TaskCaptureWikiPath(prefix, title, request string) string { - item := TaskRecord{Title: title, UUID: request} - name := path.Base(TaskNotePath("", TaskRecord{}, item)) - return strings.TrimRight(prefix, "/") + "/" + name + "--" + request + return strings.TrimRight(prefix, "/") + "/mail-" + request } func (s *Store) createCapturedTaskWiki(ctx context.Context, tx *sql.Tx, domain, actor string, item *TaskRecord, target, body string) error { diff --git a/pages-cli/pages b/pages-cli/pages index e82ce20..bf6857c 100755 --- a/pages-cli/pages +++ b/pages-cli/pages @@ -30,7 +30,7 @@ import urllib.parse as url import urllib.request as http import webbrowser -VERSION = '0.3.0' +VERSION = '0.3.1' class Failure(Exception): def __init__(self, message, status=0, payload=None): @@ -496,11 +496,24 @@ class MailText(HTMLParser): if not self.hidden: self.parts.append(data) +def mail_subject(value): + # Remove only leading mailing-list sequence markers, preserving meaningful + # tags such as [Important], and any Re:/Fwd: prefix before the marker. + marker = re.compile(r'^(?P(?:(?:re|fw|fwd)\s*:\s*)*)\[[^\[\]\r\n]+[\s:#]\s*[0-9]+\]\s*', re.IGNORECASE) + while True: + match = marker.match(value) + if not match: return value.strip() + value = match.group('reply') + value[match.end():] + + def task_mail(raw, include_body=True, override_title=None): if len(raw) > 2 << 20: raise Failure('Mail input exceeds 2 MiB; pipe one message at a time.') message = email.parser.BytesParser(policy=email.policy.default).parsebytes(raw) clean = lambda value: ' '.join(safe_text(str(value or '')).split()) - title = override_title or clean(message.get('Subject')) or '(No subject)' + subject = clean(message.get('Subject')) + title = override_title or mail_subject(subject) or '(No subject)' + if not override_title and len(title.encode()) > 300: + title = title.encode()[:297].decode('utf-8',errors='ignore').rstrip() + '…' if len(title.encode()) > 300: raise Failure('Mail subject exceeds 300 UTF-8 bytes; use --title.') notes = '\n'.join(key+': '+clean(message.get(key)) for key in ('From','Date','Message-ID') if message.get(key)) wiki_body = None @@ -513,7 +526,8 @@ def task_mail(raw, include_body=True, override_title=None): if not isinstance(content,str): raise Failure('Mail body is not readable text.') if part.get_content_type() == 'text/html': parser = MailText();parser.feed(content);parser.close();content = ''.join(parser.parts).strip() - wiki_body = notes + '\n\n' + content + # Retain the original subject when its mailing-list marker was removed. + wiki_body = (('Subject: '+subject+'\n') if title != subject else '') + notes + '\n\n' + content if '\0' in wiki_body or len(wiki_body.encode()) > 256 << 10: raise Failure('Mail Wiki body must be valid text of at most 256 KiB; nothing was uploaded.') if len(notes.encode()) > 16384: raise Failure('Mail metadata exceeds 16 KiB.') return title, notes, clean(message.get('Message-ID')) or hashlib.sha256(raw).hexdigest(), wiki_body diff --git a/pages-cli/test_productivity.py b/pages-cli/test_productivity.py index 2306bf0..26f8ed4 100644 --- a/pages-cli/test_productivity.py +++ b/pages-cli/test_productivity.py @@ -16,9 +16,27 @@ class ProductivityTests(unittest.TestCase): self.assertIn('PRIVATE BODY',body) self.assertIsNone(m['task_mail'](raw,False)[3]) raw=raw.replace(b'Subject: Test',b'Subject: '+b'a'*301) - with self.assertRaises(m['Failure']):m['task_mail'](raw) + self.assertLessEqual(len(m['task_mail'](raw)[0].encode()),300) + with self.assertRaises(m['Failure']):m['task_mail'](raw,True,'x'*301) self.assertEqual(m['task_mail'](raw,False,'Override')[0],'Override') + def test_japanese_subject_and_mailing_list_markers(self): + from email.message import EmailMessage + from email.header import Header + for encoding in ('utf-8','iso-2022-jp'): + subject='Re: [hogehoge 00123] [team:456] 日本語の依頼 [重要]' + raw=('Subject: '+Header(subject,encoding).encode()+'\r\nMessage-ID: \r\n\r\nbody').encode('ascii') + title,notes,ident,body=m['task_mail'](raw) + self.assertEqual(title,'Re: 日本語の依頼 [重要]') + self.assertEqual(ident,'');self.assertIn('Subject: '+subject,body) + for subject in ('[重要] 日本語','報告 [list 123]','[release 1.2] 更新'): + self.assertEqual(m['mail_subject'](subject),subject) + msg=EmailMessage();msg['Subject']='[list 123] '+('日本語'*100);msg.set_content('Body') + title,_,_,body=m['task_mail'](msg.as_bytes()) + self.assertLessEqual(len(title.encode()),300);self.assertTrue(title.endswith('…')) + self.assertIn('日本語'*100,body) + self.assertEqual(m['task_mail'](msg.as_bytes(),True,'[list 123] 手動指定')[0],'[list 123] 手動指定') + def test_html_mail_and_limits(self): raw=b'Subject: HTML\r\nContent-Type: text/html; charset=utf-8\r\n\r\n

Hello

World & friends

' body=m['task_mail'](raw)[3] diff --git a/pages-cli/test_task_integration.py b/pages-cli/test_task_integration.py index 7a6211d..d9ef35d 100644 --- a/pages-cli/test_task_integration.py +++ b/pages-cli/test_task_integration.py @@ -7,6 +7,7 @@ import runpy import sys from types import SimpleNamespace from unittest.mock import patch +from email.message import EmailMessage def run(): m=runpy.run_path(str(Path(__file__).with_name('pages'))) @@ -29,15 +30,19 @@ def run(): report=task('report','--json');assert sum(d['points'] for d in report['days'])==7 assert any(i['uuid']==ident for i in report['tasks']) task('preferences','--wiki-prefix','/private/mail') - mail=b'Subject: CLI mail\r\nMessage-ID: \r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPrivate mail body {{children}}' + message=EmailMessage();message['Subject']='[hogehoge 123] 日本語の件名と依頼';message['Message-ID']='';message.set_content('Private mail body {{children}}') + mail=message.as_bytes() with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))): captured=task('from-mail') - assert captured['item']['page_uuid'] and captured['page_path'].startswith('/private/mail/CLI mail--') + assert captured['item']['page_uuid'] and captured['page_path']=='/private/mail/mail-'+captured['item']['uuid'] + assert captured['item']['title']=='日本語の件名と依頼' assert 'Private mail body' not in captured['item']['notes'] with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))): assert task('from-mail')['item']['page_uuid']==captured['item']['page_uuid'] note=client.json(m['TASK_API']+'items/'+captured['item']['uuid']+'/note') assert note['linked'] and note['path']==captured['page_path'] + raw,_=client.raw('path:'+captured['page_path']) + assert '日本語の件名と依頼' in raw.decode() and 'Private mail body' in raw.decode() path=Path(fixture['root'])/'person.vcf' path.write_text('BEGIN:VCARD\r\nVERSION:4.0\r\nUID:urn:test:cli\r\nFN:CLI Person\r\nEMAIL:cli@example.invalid\r\nEND:VCARD\r\n') output=io.StringIO()