feat: atomically capture task mail into Wiki and persist destination defaults
This commit is contained in:
1 parent
52b501d1ce
commit
80d297dab4
14 files changed
+525
-30
No files matched your search
+63
-18
@@ -10,6 +10,7 @@ import difflib
|
||||
import fcntl
|
||||
import getpass
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -17,6 +18,7 @@ import posixpath
|
||||
import re
|
||||
import secrets
|
||||
import shlex
|
||||
import ssl
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -28,6 +30,7 @@ import urllib.parse as url
|
||||
import urllib.request as http
|
||||
import webbrowser
|
||||
|
||||
VERSION = '0.3.0'
|
||||
|
||||
class Failure(Exception):
|
||||
def __init__(self, message, status=0, payload=None):
|
||||
@@ -74,14 +77,23 @@ class NoRedirect(http.HTTPRedirectHandler):
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base, token='', lang='auto'):
|
||||
def __init__(self, base, token='', lang='auto', *, insecure=False):
|
||||
self.base, self.token, self.lang = origin(base), token, lang
|
||||
self.opener = http.build_opener(NoRedirect())
|
||||
self.insecure = insecure
|
||||
handlers = [NoRedirect()]
|
||||
if insecure:
|
||||
# This client's TLS policy only; never change Python's global defaults.
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
context.set_alpn_protocols(['http/1.1'])
|
||||
handlers.append(http.HTTPSHandler(context=context))
|
||||
self.opener = http.build_opener(*handlers)
|
||||
|
||||
def request(self, route, method='GET', data=None, form=None, limit=4 << 20, etag=None, content_type=None):
|
||||
if not route.startswith('/') or route.startswith('//'):
|
||||
raise Failure('Invalid API route.')
|
||||
headers = {'Accept': 'application/json', 'User-Agent': 'pages-cli/0.1'}
|
||||
headers = {'Accept': 'application/json', 'User-Agent': 'pages-cli/' + VERSION}
|
||||
if etag is not None:
|
||||
headers['If-Match'] = etag
|
||||
if self.token:
|
||||
@@ -469,21 +481,42 @@ def task_project(client, value):
|
||||
return matches[0]
|
||||
|
||||
|
||||
def task_mail(raw, include_body=False, override_title=None):
|
||||
class MailText(HTMLParser):
|
||||
"""Extract inert text; never load links, images, styles or attachments."""
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts, self.hidden = [], []
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag in ('script','style','head'): self.hidden.append(tag)
|
||||
if not self.hidden and tag in ('br','p','div','li','tr','h1','h2','h3','pre'): self.parts.append('\n')
|
||||
def handle_endtag(self, tag):
|
||||
if self.hidden and tag == self.hidden[-1]: self.hidden.pop()
|
||||
if not self.hidden and tag in ('p','div','li','tr','h1','h2','h3','pre'): self.parts.append('\n')
|
||||
def handle_data(self, data):
|
||||
if not self.hidden: self.parts.append(data)
|
||||
|
||||
|
||||
def task_mail(raw, include_body=True, override_title=None):
|
||||
if len(raw) > 2 << 20: raise Failure('Mail input exceeds 2 MiB; pipe one message at a time.')
|
||||
message = email.parser.BytesParser(policy=email.policy.default).parsebytes(raw)
|
||||
clean = lambda value: ' '.join(safe_text(str(value or '')).split())
|
||||
title = override_title or clean(message.get('Subject')) or '(No subject)'
|
||||
if len(title.encode()) > 300: raise Failure('Mail subject exceeds 300 UTF-8 bytes; use --title.')
|
||||
notes = '\n'.join(key+': '+clean(message.get(key)) for key in ('From','Date','Message-ID') if message.get(key))
|
||||
wiki_body = None
|
||||
if include_body:
|
||||
part = message.get_body(preferencelist=('plain',))
|
||||
content = ''
|
||||
part = message.get_body(preferencelist=('plain','html'))
|
||||
if part:
|
||||
content = part.get_content()
|
||||
try: content = part.get_content()
|
||||
except (LookupError, UnicodeError, ValueError) as e: raise Failure('Mail body encoding is not supported; convert to UTF-8 first.') from e
|
||||
if not isinstance(content,str): raise Failure('Mail body is not readable text.')
|
||||
notes += '\n\n' + content
|
||||
if len(notes.encode()) > 16384: raise Failure('Mail notes exceed 16 KiB; omit --include-body.')
|
||||
return title, notes, clean(message.get('Message-ID')) or hashlib.sha256(raw).hexdigest()
|
||||
if part.get_content_type() == 'text/html':
|
||||
parser = MailText();parser.feed(content);parser.close();content = ''.join(parser.parts).strip()
|
||||
wiki_body = notes + '\n\n' + content
|
||||
if '\0' in wiki_body or len(wiki_body.encode()) > 256 << 10: raise Failure('Mail Wiki body must be valid text of at most 256 KiB; nothing was uploaded.')
|
||||
if len(notes.encode()) > 16384: raise Failure('Mail metadata exceeds 16 KiB.')
|
||||
return title, notes, clean(message.get('Message-ID')) or hashlib.sha256(raw).hexdigest(), wiki_body
|
||||
|
||||
|
||||
def tasks(client, profile, words):
|
||||
@@ -497,7 +530,9 @@ def tasks(client, profile, words):
|
||||
p=sub.add_parser(name)
|
||||
if name=='add':p.add_argument('title')
|
||||
elif name=='update':p.add_argument('uuid',type=task_uuid);p.add_argument('--title')
|
||||
else:p.add_argument('--title');p.add_argument('--include-body',action='store_true')
|
||||
else:
|
||||
p.add_argument('--title');p.add_argument('--wiki-prefix',help='Wiki parent Path (otherwise use personal default or project Wiki subtree)')
|
||||
body=p.add_mutually_exclusive_group();body.add_argument('--include-body',action='store_true',help='Save body to Wiki (default; retained for compatibility)');body.add_argument('--metadata-only',action='store_true',help='Do not upload body or create a Wiki document')
|
||||
p.add_argument('--notes');p.add_argument('--project');p.add_argument('--assignee')
|
||||
p.add_argument('--due');p.add_argument('--start-on');p.add_argument('--planned');p.add_argument('--review')
|
||||
p.add_argument('--estimate',type=int);p.add_argument('--complexity',type=int,choices=range(1,6))
|
||||
@@ -507,7 +542,7 @@ def tasks(client, profile, words):
|
||||
sub.add_parser('stop');sub.add_parser('status')
|
||||
p=sub.add_parser('retry');p.add_argument('request_id',type=task_uuid)
|
||||
p=sub.add_parser('report');p.add_argument('--from',dest='start');p.add_argument('--to',dest='end')
|
||||
p=sub.add_parser('preferences');p.add_argument('--project');p.add_argument('--assignee');p.add_argument('--due-days',type=int);p.add_argument('--start-days',type=int);p.add_argument('--planned-days',type=int);p.add_argument('--review-days',type=int);p.add_argument('--estimate',type=int);p.add_argument('--complexity',type=int);p.add_argument('--digest',choices=['on','off']);p.add_argument('--at');p.add_argument('--timezone')
|
||||
p=sub.add_parser('preferences');p.add_argument('--wiki-prefix');p.add_argument('--project');p.add_argument('--assignee');p.add_argument('--due-days',type=int);p.add_argument('--start-days',type=int);p.add_argument('--planned-days',type=int);p.add_argument('--review-days',type=int);p.add_argument('--estimate',type=int);p.add_argument('--complexity',type=int);p.add_argument('--digest',choices=['on','off']);p.add_argument('--at');p.add_argument('--timezone')
|
||||
for child in sub.choices.values():child.add_argument('--json',action='store_true')
|
||||
a=parser.parse_args(words);action=a.action
|
||||
def emit(data): print(json.dumps(data,ensure_ascii=False,indent=2))
|
||||
@@ -516,8 +551,10 @@ def tasks(client, profile, words):
|
||||
if len(pending)>=20 and payload['request_id'] not in pending:raise Failure('20 pending captures retained. Retry them before adding more.')
|
||||
pending[payload['request_id']]=payload;profile.save()
|
||||
try: result=client.json(TASK_API+'capture',method='POST',data=payload)
|
||||
except Failure:
|
||||
print('Draft retained; retry with: pages task retry '+payload['request_id'],file=sys.stderr);raise
|
||||
except Failure as e:
|
||||
print('Draft retained; retry with: pages task retry '+payload['request_id'],file=sys.stderr)
|
||||
if e.payload.get('error') == 'wiki_prefix_required': raise Failure('Set a private Wiki destination first: pages task preferences --wiki-prefix /YOUR/MAIL/PATH',e.status) from e
|
||||
raise
|
||||
del pending[payload['request_id']];profile.save();return result
|
||||
if action in ('list','projects'):
|
||||
query={'kind':'project' if action=='projects' else 'task','view':getattr(a,'view','all'),'limit':100,'q':getattr(a,'q',''),'before':getattr(a,'before','')}
|
||||
@@ -531,7 +568,7 @@ def tasks(client, profile, words):
|
||||
if action=='show':emit(client.json(TASK_API+'items/'+a.uuid));return
|
||||
if action=='preferences':
|
||||
data=client.json(TASK_API+'preferences');p=data['preferences'];changed=False
|
||||
for attr,key in [('project','project_uuid'),('assignee','assignee_uuid'),('due_days','due_days'),('start_days','start_days'),('planned_days','planned_days'),('review_days','review_days'),('estimate','estimate_minutes'),('complexity','complexity'),('at','digest_time'),('timezone','timezone')]:
|
||||
for attr,key in [('wiki_prefix','wiki_prefix'),('project','project_uuid'),('assignee','assignee_uuid'),('due_days','due_days'),('start_days','start_days'),('planned_days','planned_days'),('review_days','review_days'),('estimate','estimate_minutes'),('complexity','complexity'),('at','digest_time'),('timezone','timezone')]:
|
||||
value=getattr(a,attr)
|
||||
if value is not None:p[key]=task_project(client,value) if attr=='project' else value;changed=True
|
||||
if a.digest:p['digest_enabled']=a.digest=='on';changed=True
|
||||
@@ -543,8 +580,11 @@ def tasks(client, profile, words):
|
||||
if action in ('add','from-mail'):
|
||||
payload={'request_id':getattr(a,'request_id',None) or str(uuid.uuid4()),'title':a.title,'notes':a.notes or ''}
|
||||
if action=='from-mail':
|
||||
title,notes,message_id=task_mail(sys.stdin.buffer.read((2<<20)+1),a.include_body,a.title)
|
||||
if a.metadata_only and a.wiki_prefix is not None: raise Failure('--wiki-prefix cannot be combined with --metadata-only.')
|
||||
title,notes,message_id,wiki_body=task_mail(sys.stdin.buffer.read((2<<20)+1),not a.metadata_only,a.title)
|
||||
payload.update(title=a.title or title,notes=a.notes if a.notes is not None else notes,request_id=str(uuid.uuid5(uuid.NAMESPACE_URL,client.base+'\n'+profile.data.get('username','')+'\n'+message_id)))
|
||||
if wiki_body is not None: payload['wiki_body']=wiki_body
|
||||
if a.wiki_prefix is not None: payload['wiki_prefix']=a.wiki_prefix
|
||||
for attr,key in [('project','project_uuid'),('assignee','assignee_uuid'),('due','due_on'),('start_on','start_on'),('planned','planned_on'),('review','review_on'),('estimate','estimate_minutes'),('complexity','complexity')]:
|
||||
value=getattr(a,attr)
|
||||
if value is not None:payload[key]=task_project(client,value) if attr=='project' else value
|
||||
@@ -670,7 +710,7 @@ def execute(client, profile, words):
|
||||
route = addr[5:] if addr.startswith('path:') else '/o/' + addr if addr.startswith('uuid:') else '/r/view/' + addr[4:]
|
||||
webbrowser.open(client.base + url.quote(route, safe='/:'), new=2)
|
||||
elif cmd == 'edit':
|
||||
editing = Client(client.base, client.token, language(args.lang)) if args.lang else client
|
||||
editing = Client(client.base, client.token, language(args.lang), insecure=client.insecure) if args.lang else client
|
||||
editor(editing, addr, args.format)
|
||||
elif cmd == 'put':
|
||||
state = client.snapshot(addr)
|
||||
@@ -739,13 +779,16 @@ def shell(client, profile):
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(prog='pages', description=__doc__, epilog='Commands: configure, login, logout, status, shell, ls, cd, pwd, cat, view, get, put, edit, search, open, task, contacts. Use pages COMMAND --help.')
|
||||
parser.add_argument('--version', action='version', version='pages-cli ' + VERSION)
|
||||
parser.add_argument('--profile', help='Use another configured connection for this invocation')
|
||||
parser.add_argument('--lang', help='auto or an exact language tag for this invocation')
|
||||
parser.add_argument('-k', '--insecure', action='store_true', help='Skip TLS certificate and hostname verification for this invocation only (unsafe; place before COMMAND)')
|
||||
parser.add_argument('command', nargs='?', default='shell')
|
||||
parser.add_argument('args', nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == 'configure':
|
||||
if args.profile or args.lang: parser.error('Use pages configure set --profile NAME --lang TAG.')
|
||||
if args.insecure: parser.error('--insecure is invocation-only; configure does not connect or save a TLS bypass.')
|
||||
configure(args.args); return
|
||||
if args.command == 'help':
|
||||
parser.print_help(); return
|
||||
@@ -767,10 +810,12 @@ def main(argv=None):
|
||||
lang = language(args.lang or spec.get('lang', 'auto'))
|
||||
# Hold the profile lock before releasing the configuration lock.
|
||||
profile = stack.enter_context(Profile(name).locked())
|
||||
if args.insecure:
|
||||
print('WARNING: --insecure disables TLS certificate and hostname verification for this invocation. Credentials and data can be intercepted; this option is not saved.', file=sys.stderr)
|
||||
if args.command == 'login':
|
||||
if not login_args.oidc and not spec.get('username'): raise Failure('Set your account with pages configure --username NAME, or use pages login --oidc.')
|
||||
if profile.data.get('token'): raise Failure('Log out this profile before replacing its session.')
|
||||
client = Client(spec['origin'], lang=lang)
|
||||
client = Client(spec['origin'], lang=lang, insecure=args.insecure)
|
||||
if login_args.oidc:
|
||||
expires, username = client.login_oidc(login_args.no_browser, spec.get('username', ''))
|
||||
else:
|
||||
@@ -799,7 +844,7 @@ def main(argv=None):
|
||||
if args.command != 'logout' and (profile.data.get('origin') != spec['origin'] or profile.data.get('configured_username', profile.data.get('username')) != spec.get('username', '')):
|
||||
raise Failure('Session and configured identity differ. Log out before reconnecting.')
|
||||
if args.command not in ('logout', 'status') and profile.data.get('expires', 0) <= time.time(): raise Failure('Session expired; log out and log in again.')
|
||||
client = Client(profile.data['origin'], profile.data['token'], lang)
|
||||
client = Client(profile.data['origin'], profile.data['token'], lang, insecure=args.insecure)
|
||||
if args.command == 'shell': shell(client, profile)
|
||||
else: execute(client, profile, [args.command] + args.args)
|
||||
|
||||
|
||||
@@ -11,13 +11,43 @@ m=runpy.run_path(str(Path(__file__).with_name('pages')))
|
||||
class ProductivityTests(unittest.TestCase):
|
||||
def test_mail_privacy_and_long_subject_override(self):
|
||||
raw=b'Subject: Test\r\nFrom: Person <person@example.invalid>\r\nMessage-ID: <id@example.invalid>\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPRIVATE BODY'
|
||||
title,notes,ident=m['task_mail'](raw)
|
||||
title,notes,ident,body=m['task_mail'](raw)
|
||||
self.assertEqual(title,'Test');self.assertNotIn('PRIVATE BODY',notes);self.assertIn('Message-ID:',notes)
|
||||
self.assertIn('PRIVATE BODY',m['task_mail'](raw,True)[1])
|
||||
self.assertIn('PRIVATE BODY',body)
|
||||
self.assertIsNone(m['task_mail'](raw,False)[3])
|
||||
raw=raw.replace(b'Subject: Test',b'Subject: '+b'a'*301)
|
||||
with self.assertRaises(m['Failure']):m['task_mail'](raw)
|
||||
self.assertEqual(m['task_mail'](raw,False,'Override')[0],'Override')
|
||||
|
||||
def test_html_mail_and_limits(self):
|
||||
raw=b'Subject: HTML\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<head><style>secret</style></head><p>Hello</p><script>evil()</script><img src="https://example.invalid/image"><p>World & friends</p>'
|
||||
body=m['task_mail'](raw)[3]
|
||||
self.assertIn('Hello\n',body);self.assertIn('World & friends',body)
|
||||
for value in ('secret','evil','https://'):self.assertNotIn(value,body)
|
||||
for body in (b'a'*(256<<10),b'bad\0text'):
|
||||
with self.assertRaises(m['Failure']):m['task_mail'](b'Subject: Test\r\n\r\n'+body)
|
||||
|
||||
def test_mail_payload_wiki_and_metadata_only(self):
|
||||
from unittest.mock import patch
|
||||
raw=b'Subject: Test\r\nMessage-ID: <same@example.invalid>\r\n\r\nPRIVATE BODY'
|
||||
calls=[]
|
||||
client=SimpleNamespace(base='https://example.invalid',json=lambda route,**kw:calls.append(kw['data'].copy()) or {'item':{'uuid':kw['data']['request_id']}})
|
||||
profile=SimpleNamespace(data={'username':'test'},save=lambda:None)
|
||||
for args in (['--wiki-prefix','/private/mail'],['--metadata-only']):
|
||||
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(raw))),contextlib.redirect_stdout(io.StringIO()):
|
||||
m['tasks'](client,profile,['from-mail',*args])
|
||||
self.assertIn('PRIVATE BODY',calls[0]['wiki_body']);self.assertNotIn('PRIVATE BODY',calls[0]['notes'])
|
||||
self.assertEqual(calls[0]['wiki_prefix'],'/private/mail')
|
||||
self.assertNotIn('wiki_body',calls[1]);self.assertEqual(calls[0]['request_id'],calls[1]['request_id'])
|
||||
|
||||
def test_multipart_prefers_plain_and_skips_attachment(self):
|
||||
from email.message import EmailMessage
|
||||
msg=EmailMessage();msg['Subject']='Mixed';msg.set_content('Plain body')
|
||||
msg.add_alternative('<p>HTML alternate</p>',subtype='html')
|
||||
msg.add_attachment(b'ATTACHMENT',maintype='application',subtype='octet-stream',filename='secret.bin')
|
||||
body=m['task_mail'](msg.as_bytes())[3]
|
||||
self.assertIn('Plain body',body);self.assertNotIn('alternate',body);self.assertNotIn('ATTACHMENT',body)
|
||||
|
||||
def test_capture_retains_request_until_explicit_retry(self):
|
||||
calls=[]
|
||||
def send(route,**kwargs):
|
||||
@@ -42,3 +72,12 @@ class ProductivityTests(unittest.TestCase):
|
||||
for cmd in [('task','--help'),('contacts','--help'),('task','add','--help')]:
|
||||
with contextlib.redirect_stdout(io.StringIO()),self.assertRaises(SystemExit) as raised:m['main'](list(cmd))
|
||||
self.assertEqual(raised.exception.code,0)
|
||||
|
||||
def test_neomutt_query_leading_dash_is_data(self):
|
||||
calls=[]
|
||||
def query(route):
|
||||
calls.append(route)
|
||||
return {'items':[{'name':'Person','emails':['person@example.invalid'],'organization':''}]}
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
m['contacts'](SimpleNamespace(json=query),['query','--neomutt','--','--unexpected'])
|
||||
self.assertIn('q=--unexpected',calls[0])
|
||||
@@ -5,6 +5,8 @@ import json
|
||||
from pathlib import Path
|
||||
import runpy
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
def run():
|
||||
m=runpy.run_path(str(Path(__file__).with_name('pages')))
|
||||
@@ -26,6 +28,16 @@ def run():
|
||||
task('done',ident)
|
||||
report=task('report','--json');assert sum(d['points'] for d in report['days'])==7
|
||||
assert any(i['uuid']==ident for i in report['tasks'])
|
||||
task('preferences','--wiki-prefix','/private/mail')
|
||||
mail=b'Subject: CLI mail\r\nMessage-ID: <integration@example.invalid>\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPrivate mail body {{children}}'
|
||||
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))):
|
||||
captured=task('from-mail')
|
||||
assert captured['item']['page_uuid'] and captured['page_path'].startswith('/private/mail/CLI mail--')
|
||||
assert 'Private mail body' not in captured['item']['notes']
|
||||
with patch('sys.stdin',SimpleNamespace(buffer=io.BytesIO(mail))):
|
||||
assert task('from-mail')['item']['page_uuid']==captured['item']['page_uuid']
|
||||
note=client.json(m['TASK_API']+'items/'+captured['item']['uuid']+'/note')
|
||||
assert note['linked'] and note['path']==captured['page_path']
|
||||
path=Path(fixture['root'])/'person.vcf'
|
||||
path.write_text('BEGIN:VCARD\r\nVERSION:4.0\r\nUID:urn:test:cli\r\nFN:CLI Person\r\nEMAIL:cli@example.invalid\r\nEND:VCARD\r\n')
|
||||
output=io.StringIO()
|
||||
|
||||
Reference in new issue
Block a user