convert internal message format from json to protobuf

6 files changed, 547 insertions, 234 deletions

diff --git a/.gitignore b/.gitignore
index fca7333..1094ba8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,5 @@ user_sessions
 src/jwebmail/static/css/
 dist/
 jwebmail.prod.toml
+jwebmail_pb2.py
+jwebmail_pb2.pyi
diff --git a/pyproject.toml b/pyproject.toml
index 47d8692..cb9caf6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,7 @@ dependencies = [
   "flask-paginate",
   "email-validator",
   "redis",
+  "protobuf",
 ]
 
 [build-system]
@@ -33,8 +34,10 @@ dependencies = [
 
 [tool.hatch.envs.dev.scripts]
 serve = "flask --app src/jwebmail --debug run --extra-files src/jwebmail/translations"
+server = "env JWEBMAIL_CONFIG=../../jwebmail.toml flask --app src/jwebmail run"
 tr-compile = "pybabel compile -d src/jwebmail/translations/"
 tr-extract = "pybabel extract -F babel.cfg -o messages.pot -k lazy_gettext src/ && pybabel update -i messages.pot -d src/jwebmail/translations/"
+pb-generate = "protoc -I src/jwebmail/model --python_out=src/jwebmail/model/ jwebmail.proto"
 
 [tool.hatch.build.targets.wheel]
 ignore-vcs = true
diff --git a/script/extract.py b/script/extract.py
index e771110..5bbbec8 100755
--- a/script/extract.py
+++ b/script/extract.py
@@ -7,8 +7,10 @@ Runs with elevated privileges.
 
 This program is started by qmail-authuser with elevated privileges after
 a successful login.
-Input directives are provided as command line arguments.
-Output is delivered via STDOUT as json and log information via STDERR.
+
+The run method is provided by a command line argument.
+Additional data is read from STDIN as protobuf.
+Output is delivered via STDOUT as protobuf and log information via STDERR.
 
 Exit codes::
 
@@ -23,7 +25,6 @@ Exit codes::
 
 import email.parser
 import email.policy
-import json
 import logging
 import re
 from argparse import ArgumentParser
@@ -36,7 +37,9 @@ from os import environ, getpid, path, setuid
 from pathlib import Path
 from pwd import getpwnam
 from sys import exit as sysexit
-from sys import stdout
+from sys import stdin, stdout
+
+import jwebmail.model.jwebmail_pb2 as jwebmail
 
 
 class MyMaildir(Maildir):
@@ -64,7 +67,8 @@ class MyMaildir(Maildir):
 
     def _refresh(self):
         super()._refresh()
-        for r in list(k for k in self._toc if k.startswith(".")):
+        rm = [r for r in self._toc if r.startswith(".")]
+        for r in rm:
             del self._toc[r]
 
 
@@ -76,9 +80,12 @@ class QMAuthError(Exception):
 
 def _adr(addrs):
     if addrs is None:
-        return None
+        return []
     return [
-        {"address": addr.addr_spec, "display_name": addr.display_name}
+        jwebmail.MailHeader.MailAddr(
+            address=addr.addr_spec,
+            name=addr.display_name,
+        )
         for addr in addrs.addresses
     ]
 
@@ -103,9 +110,9 @@ def startup(maildir, su, user, mode):
         sysexit(5)
 
     def create_messages(mail_file):
-        if mode == count_mails:
+        if mode == "count":
             msg = MaildirMessage(None)
-        elif mode == list_mails:
+        elif mode == "list":
             msg = MaildirMessage(
                 email.parser.BytesHeaderParser(policy=email.policy.default).parse(
                     mail_file
@@ -160,77 +167,107 @@ def _sort_mails(f, sort):
 
 
 def _get_mime_head_info(msg):
-    return {
-        "content_maintype": msg.get_content_maintype(),
-        "content_subtype": msg.get_content_subtype(),
-        "content_disposition": msg.get_content_disposition(),
-        "filename": msg.get_filename(),
-    }
+    mh = jwebmail.MIMEHeader(
+        maintype=msg.get_content_maintype(),
+        subtype=msg.get_content_subtype(),
+    )
+    if (cd := msg.get_content_disposition()) == "inline":
+        mh.contentdispo = (
+            jwebmail.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_INLINE
+        )
+    elif cd == "attachment":
+        mh.contentdispo = (
+            jwebmail.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
+        )
+    elif cd is None:
+        mh.contentdispo = (
+            jwebmail.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_NONE
+        )
+    else:
+        assert False
+
+    if fn := msg.get_filename():
+        mh.file_name = fn
+
+    return mh
 
 
 def _get_head_info(msg):
-    return {
-        "date": msg["date"].datetime.isoformat(),
-        "from": _adr(msg["from"]),
-        "sender": _adr(msg["sender"]),
-        "reply_to": _adr(msg["reply-to"]),
-        "to": _adr(msg["to"]),
-        "cc": _adr(msg["cc"]),
-        "bcc": _adr(msg["bcc"]),
-        "subject": msg["subject"],
-        "comments": msg["comments"],
-        "keywords": msg["keywords"],
-        "mime": _get_mime_head_info(msg),
-    }
-
-
-def list_mails(f, start, end, sortby, folder):
-    assert 0 <= start <= end
-
-    if folder:
-        f = f.get_folder(folder)
-
-    if start == end:
+    mh = jwebmail.MailHeader(
+        send_date=msg["date"].datetime.isoformat(),
+        written_from=_adr(msg["from"]),
+        reply_to=_adr(msg["reply-to"]),
+        send_to=_adr(msg["to"]),
+        cc=_adr(msg["cc"]),
+        bcc=_adr(msg["bcc"]),
+        subject=msg["subject"],
+        comments=msg["comments"],
+        keywords=msg["keywords"],
+        mime=_get_mime_head_info(msg),
+    )
+
+    if s := _adr(msg["sender"]):
+        mh.sender = s[0]
+
+    return mh
+
+
+def list_mails(f, req):
+    r = jwebmail.ListReq()
+    r.ParseFromString(req)
+
+    assert 0 <= r.start <= r.end
+
+    if r.folder:
+        f = f.get_folder(r.folder)
+
+    if r.start == r.end:
         return []
 
-    kfn, reverse = _sort_mails(f, sortby)
+    kfn, reverse = _sort_mails(f, r.sort)
     msgs = list(f.items())
     msgs.sort(key=kfn, reverse=reverse)
-    msgs = msgs[start : min(len(msgs), end)]
-
-    return [
-        {
-            "message_handle": mid,
-            "byte_size": path.getsize(f.get_filename(mid)),
-            "unread": "S" in msg.get_flags(),
-            "date_received": datetime.fromtimestamp(_get_rcv_time(mid)).isoformat(),
-            "head": _get_head_info(msg),
-        }
+    msgs = msgs[r.start : min(len(msgs), r.end)]
+
+    items = [
+        jwebmail.ListMailHeader(
+            mid=mid,
+            byte_size=path.getsize(f.get_filename(mid)),
+            unread="S" in msg.get_flags(),
+            rec_date=datetime.fromtimestamp(_get_rcv_time(mid)).isoformat(),
+            header=_get_head_info(msg),
+        )
         for mid, msg in msgs
     ]
+    return jwebmail.ListResp(mail_heads=items).SerializeToString()
 
 
-def count_mails(f, subfolder):
-    if subfolder:
-        f = f.get_folder(subfolder)
+def count_mails(f, req):
+    r = jwebmail.StatsReq()
+    r.ParseFromString(req)
+    if r.folder:
+        f = f.get_folder(r.folder)
 
-    return {
-        "total_mails": len(f),
-        "byte_size": sum(path.getsize(f.get_filename(mid)) for mid in f.keys()),
-        "unread_mails": len([1 for m in f if "S" in m.get_flags()]),
-    }
+    resp = jwebmail.StatsResp(
+        mail_count=len(f),
+        unread_count=len([1 for m in f if "S" in m.get_flags()]),
+        byte_size=sum(path.getsize(f.get_filename(mid)) for mid in f.keys()),
+    )
+    return resp.SerializeToString()
 
 
 def _get_body(mail):
     if not mail.is_multipart():
         if mail.get_content_maintype() == "text":
-            return mail.get_content()
+            return jwebmail.MailBody(discrete=mail.get_content())
         else:
             ret = mail.get_content()
             if ret.isascii():
-                return ret.decode(encoding="ascii")
+                return jwebmail.MailBody(discrete=ret.decode(encoding="ascii"))
             elif len(ret) <= 128 * 1024:
-                return b64encode(ret).decode(encoding="ascii")
+                return jwebmail.MailBody(
+                    discrete=b64encode(ret).decode(encoding="ascii")
+                )
             else:
                 raise QMAuthError(
                     "non attachment part too large (>512kB)", size=len(ret)
@@ -238,105 +275,102 @@ def _get_body(mail):
 
     if (mctype := mail.get_content_maintype()) == "message":
         msg = mail.get_content()
-        return {
-            "head": _get_head_info(msg),
-            "body": _get_body(msg),
-        }
+        return jwebmail.MailBody(
+            mail=jwebmail.Mail(head=_get_head_info(msg), body=_get_body(msg))
+        )
     elif mctype == "multipart":
-        ret = {
-            "preamble": mail.preamble,
-            "parts": [],
-            "epilogue": mail.epilogue,
-        }
+        ret = jwebmail.MailBody.Multipart(
+            preamble=mail.preamble,
+            epilogue=mail.epilogue,
+        )
         for part in mail.iter_parts():
             head = _get_mime_head_info(part)
-            if head["content_disposition"] != "attachment":
+            if (
+                head.contentdispo
+                != jwebmail.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
+            ):
                 body = _get_body(part)
             else:
                 body = None
-            ret["parts"].append(
-                {
-                    "head": head,
-                    "body": body,
-                }
+            ret.parts.append(
+                jwebmail.MIMEPart(
+                    mime_header=head,
+                    body=body,
+                )
             )
-        return ret
+        return jwebmail.MailBody(multipart=ret)
     else:
         raise ValueError(f"unknown major content-type {mctype!r}")
 
 
-def read_mail(f, subfolder, mid):
-    if subfolder:
-        f = f.get_folder(subfolder)
+def read_mail(f, req):
+    r = jwebmail.ShowReq()
+    r.ParseFromString(req)
+
+    if r.folder:
+        f = f.get_folder(r.folder)
 
-    msg = f.get(mid, None)
+    msg = f.get(r.mid, None)
     if not msg:
-        raise QMAuthError("no such message", mid=mid)
+        raise QMAuthError("no such message", mid=r.mid)
 
-    return {
-        "head": _get_head_info(msg),
-        "body": _get_body(msg),
-    }
+    res = jwebmail.Mail(
+        head=_get_head_info(msg),
+        body=_get_body(msg),
+    )
+    return jwebmail.ShowResp(mail=res).SerializeToString()
 
 
 def _descent(xx):
     head = _get_mime_head_info(xx)
-    if (mctype := head["content_maintype"]) == "message":
+    if (mctype := head.maintype) == "message":
         body = xx.get_content()
     elif mctype == "multipart":
         body = xx.iter_parts()
     else:
         body = xx.get_content()
-    return {
-        "head": head,
-        "body": body,
-    }
+    return head, body
+
 
+def raw_mail(f, req):
+    r = jwebmail.RawReq()
+    r.ParseFromString(req)
 
-def raw_mail(f, subfolder, mid, path):
-    if subfolder:
-        f = f.get_folder(subfolder)
+    if r.folder:
+        f = f.get_folder(r.folder)
 
-    msg = f.get(mid, None)
+    msg = f.get(r.mid, None)
     if not msg:
-        raise QMAuthError("no such message", mid=mid)
+        raise QMAuthError("no such message", mid=r.mid)
 
-    pth = [int(seg) for seg in path.split(".")] if path else []
-    mail = {
-        "head": {"content_maintype": "message", "content_subtype": "rfc822"},
-        "body": msg,
-    }
+    pth = [int(seg) for seg in r.path.split(".")] if r.path else []
+    h = jwebmail.MIMEHeader(maintype="message", subtype="rfc822")
+    b = msg
 
     for n in pth:
-        mctype = mail["head"]["content_maintype"]
+        mctype = h.maintype
 
         if mctype == "multipart":
             try:
-                res = next(islice(mail["body"], n, None))
+                res = next(islice(b, n, None))
             except StopIteration:
                 raise QMAuthError("out of bounds path for mail", path=pth)
-            mail = _descent(res)
+            (h, b) = _descent(res)
         elif mctype == "message":
             assert n == 0
-            mail = _descent(mail["body"])
+            (h, b) = _descent(b)
         else:
             raise QMAuthError(
                 f"can not descent into non multipart content type {mctype}"
             )
 
-    if hasattr(mail["body"], "__next__"):
+    if hasattr(b, "__next__"):
         raise QMAuthError("can not stop at multipart section", path=pth)
 
-    json.dump(mail["head"], stdout)
-    stdout.write("\n")
-    if isinstance(mail["body"], str):
-        stdout.write(mail["body"])
-    elif isinstance(mail["body"], bytes):
-        stdout.flush()
-        stdout.buffer.write(mail["body"])
-    else:
-        stdout.write(str(mail["body"]))
-    sysexit(0)
+    if isinstance(b, str):
+        b = b.encode()
+
+    return jwebmail.RawResp(header=h, body=b).SerializeToString()
 
 
 def _matches(m, pattern):
@@ -349,51 +383,83 @@ def _matches(m, pattern):
     return re.search(pattern, m.body.decoded()) or re.search(pattern, m.subject)
 
 
-def search_mails(f, pattern: str, subfolder: str):
-    if subfolder:
-        f = f.get_folder(subfolder)
+def search_mails(f, req):
+    r = jwebmail.SearchReq()
+    r.ParseFromString(req)
 
-    return [
-        {
-            "head": _get_head_info(msg),
-            "body": _get_body(msg),
-        }
+    if r.folder:
+        f = f.get_folder(r.folder)
+
+    res = [
+        jwebmail.ListMailHeader(
+            header=_get_head_info(msg),
+        )
         for msg in f.values()
-        if _matches(msg, pattern)
+        if _matches(msg, r.pattern)
     ]
+    return jwebmail.SearchResp(found=res).SerializeToString()
 
 
-def folders(f):
-    return f.list_folders()
+def folders(f, req):
+    r = jwebmail.FoldersReq()
+    r.ParseFromString(req)
+    return jwebmail.FoldersResp(folders=f.list_folders()).SerializeToString()
 
 
-def move_mail(f, mid, from_, to):
-    if from_:
-        f = f.get_folder(from_)
+def move_mail(f, req):
+    r = jwebmail.MoveReq()
+    r.ParseFromString(req)
 
-    fname = Path(f.get_filename(mid))
+    if r.from_f:
+        f = f.get_folder(r.from_f)
 
-    assert to in f.list_folders()
+    fname = Path(f.get_filename(r.mid))
 
-    sep = -2 if not from_ else -3
+    assert r.to_f in f.list_folders()
 
-    if to:
-        res = fname.parts[:sep] + ("." + to,) + fname.parts[-2:]
+    sep = -2 if not r.from_f else -3
+
+    if r.to_f:
+        res = fname.parts[:sep] + ("." + r.to_f,) + fname.parts[-2:]
     else:
         res = fname.parts[:sep] + fname.parts[-2:]
 
     fname.rename(Path(*res))
 
-    return 1
+    return jwebmail.MoveResp().SerializeToString()
+
 
+def remove_mail(f, req):
+    r = jwebmail.RemoveReq()
+    r.ParseFromString(req)
 
-def remove_mail(f, subdir, mid):
-    if subdir:
-        f = f.get_folder(subdir)
+    if r.folder:
+        f = f.get_folder(r.folder)
 
-    f[mid].add_flag("T")
+    f[r.mid].add_flag("T")
 
-    return 1
+    return jwebmail.RemoveResp().SerializeToString()
+
+
+def method_to_run(value):
+    if value == "list":
+        return list_mails
+    elif value == "count":
+        return count_mails
+    elif value == "read":
+        return read_mail
+    elif value == "raw":
+        return raw_mail
+    elif value == "folders":
+        return folders
+    elif value == "move":
+        return move_mail
+    elif value == "remove":
+        return remove_mail
+    elif value == "search":
+        return search_mails
+    else:
+        raise ValueError(value)
 
 
 def parse_arguments():
@@ -402,48 +468,9 @@ def parse_arguments():
     ap.add_argument("os_user")
     ap.add_argument("mail_user")
 
-    sp = ap.add_subparsers(title="methods", required=True)
-
-    sp_list = sp.add_parser("list")
-    sp_list.add_argument("folder", metavar="subfolder")
-    sp_list.add_argument("start", type=int)
-    sp_list.add_argument("end", type=int)
-    sp_list.add_argument("sortby", metavar="sort_by")
-    sp_list.set_defaults(run=list_mails)
-
-    sp_count = sp.add_parser("count")
-    sp_count.add_argument("subfolder")
-    sp_count.set_defaults(run=count_mails)
-
-    sp_read = sp.add_parser("read")
-    sp_read.add_argument("subfolder")
-    sp_read.add_argument("mid", metavar="message")
-    sp_read.set_defaults(run=read_mail)
-
-    sp_raw = sp.add_parser("raw")
-    sp_raw.add_argument("subfolder")
-    sp_raw.add_argument("mid", metavar="message")
-    sp_raw.add_argument("path", default="")
-    sp_raw.set_defaults(run=raw_mail)
-
-    sp_folders = sp.add_parser("folders")
-    sp_folders.set_defaults(run=folders)
-
-    sp_move = sp.add_parser("move")
-    sp_move.add_argument("mid", metavar="message")
-    sp_move.add_argument("from_", metavar="from")
-    sp_move.add_argument("to")
-    sp_move.set_defaults(run=move_mail)
-
-    sp_remove = sp.add_parser("remove")
-    sp_remove.add_argument("subdir")
-    sp_remove.add_argument("mid", metavar="message")
-    sp_remove.set_defaults(run=remove_mail)
-
-    sp_search = sp.add_parser("search")
-    sp_search.add_argument("pattern")
-    sp_search.add_argument("subfolder")
-    sp_search.set_defaults(run=search_mails)
+    ap.add_argument(
+        "method", choices=["list", "count", "read", "raw", "folders", "move", "remove"]
+    )
 
     return vars(ap.parse_args())
 
@@ -457,19 +484,22 @@ def main():
         args = parse_arguments()
         logging.debug("started with %s", args)
         s = startup(
-            args.pop("maildir_path"),
-            args.pop("os_user"),
-            args.pop("mail_user"),
-            args["run"],
+            args["maildir_path"],
+            args["os_user"],
+            args["mail_user"],
+            args["method"],
         )
         logging.debug("setuid successful")
-        run = args.pop("run")
-        reply = run(s, **args)
-        json.dump(reply, stdout)
-    except QMAuthError as qerr:
-        errmsg = dict(error=qerr.msg, **qerr.info)
-        json.dump(errmsg, stdout)
-        sysexit(3)
+        stdout.write("OPEN\n")
+        stdout.flush()
+        val = stdin.buffer.read()
+        run = method_to_run(args["method"])
+        reply = run(s, val)
+        logging.debug("pb method(%s) size(%d)", args["method"], len(reply))
+        stdout.buffer.write(reply)
+    #except QMAuthError as qerr:
+    #    errmsg = dict(error=qerr.msg, **qerr.info)
+    #    sysexit(3)
     except Exception:
         logging.exception("qmauth.py error")
         sysexit(4)
diff --git a/script/testauthenticator.py b/script/testauthenticator.py
index 91767ad..fa7a1ac 100755
--- a/script/testauthenticator.py
+++ b/script/testauthenticator.py
@@ -9,8 +9,8 @@ VALID_PW = b"12345"
 
 
 def main():
-    with os.fdopen(3, "rb") as authfd:
-        inp = authfd.read()
+    with os.fdopen(3, "rb", buffering=0) as authfd:
+        inp = authfd.read(511)
 
     u, p, *r = inp.split(b"\0")
     if len(r) > 2:
diff --git a/src/jwebmail/model/jwebmail.proto b/src/jwebmail/model/jwebmail.proto
new file mode 100644
index 0000000..bf1454a
--- /dev/null
+++ b/src/jwebmail/model/jwebmail.proto
@@ -0,0 +1,156 @@
+syntax = "proto3";
+
+package jwebmail;
+
+message MIMEHeader {
+
+  enum ContentDisposition {
+    CONTENT_DISPOSITION_NONE       = 0;
+    CONTENT_DISPOSITION_INLINE     = 1;
+    CONTENT_DISPOSITION_ATTACHMENT = 2;
+  }
+
+  string             maintype     = 1;
+  string             subtype      = 2;
+  ContentDisposition contentdispo = 3;
+  optional string    file_name    = 4;
+}
+
+message MailHeader {
+
+  message MailAddr {
+    optional string name    = 1;
+    string          address = 2;
+  }
+
+  string            send_date    =  1;
+  repeated MailAddr written_from =  2;
+  optional MailAddr sender       =  3;
+  repeated MailAddr reply_to     =  4;
+  repeated MailAddr send_to      =  5;
+  repeated MailAddr cc           =  6;
+  repeated MailAddr bcc          =  7;
+  string            subject      =  8;
+  repeated string   comments     =  9;
+  repeated string   keywords     = 10;
+  MIMEHeader        mime         = 11;
+}
+
+message ListMailHeader {
+  uint64     byte_size  = 1;
+  bool       unread     = 2;
+  string     rec_date   = 3;
+  string     mid        = 4;
+  MailHeader header     = 5;
+}
+
+message MailBody {
+  message Multipart {
+    optional string   preamble = 1;
+    repeated MIMEPart parts    = 2;
+    optional string   epilogue = 3;
+  }
+
+  oneof Body {
+    string    discrete  = 1;
+    Multipart multipart = 2;
+    Mail      mail      = 3;
+  }
+}
+
+message Mail {
+  MailHeader head = 1;
+  MailBody   body = 2;
+}
+
+message MIMEPart {
+  MIMEHeader mime_header = 1;
+  MailBody   body        = 2;
+}
+
+// Request-Response pairs
+
+message ListReq {
+  string folder = 1;
+  int32  start  = 2;
+  int32  end    = 3;
+  string sort   = 4;
+}
+
+message ListResp {
+  repeated ListMailHeader mail_heads = 1;
+}
+
+message StatsReq {
+  string folder = 1;
+}
+
+message StatsResp {
+  uint32 mail_count   = 1;
+  uint32 unread_count = 2;
+  uint64 byte_size    = 3;
+}
+
+message ShowReq {
+  string folder = 1;
+  string mid    = 2;
+}
+
+message ShowResp {
+  Mail mail = 1;
+}
+
+message RawReq {
+  string          folder = 1;
+  string          mid    = 2;
+  optional string path   = 3;
+}
+
+message RawResp {
+  MIMEHeader header = 1;
+  bytes      body   = 2;
+}
+
+message SearchReq {
+  string folder  = 1;
+  string pattern = 2;
+}
+
+message SearchResp {
+  repeated ListMailHeader found = 1;
+}
+
+message FoldersReq {
+}
+
+message FoldersResp {
+  repeated string folders = 1;
+}
+
+message MoveReq {
+  string mid    = 1;
+  string from_f = 2;
+  string to_f   = 3;
+}
+
+message MoveResp {
+}
+
+message RemoveReq {
+  string folder = 1;
+  string mid    = 2;
+}
+
+message RemoveResp {
+}
+
+service MailService {
+  rpc List(ListReq)       returns (ListResp);
+  rpc Stats(StatsReq)     returns (StatsResp);
+  rpc Show(ShowReq)       returns (ShowResp);
+  rpc Raw(RawReq)         returns (RawResp);
+  rpc Search(SearchReq)   returns (SearchResp);
+  rpc Folders(FoldersReq) returns (FoldersResp);
+  rpc Move(MoveReq)       returns (MoveResp);
+  rpc Remove(RemoveReq)   returns (RemoveResp);
+}
diff --git a/src/jwebmail/model/read_mails.py b/src/jwebmail/model/read_mails.py
index 45d5996..8ba3c67 100644
--- a/src/jwebmail/model/read_mails.py
+++ b/src/jwebmail/model/read_mails.py
@@ -1,8 +1,9 @@
-import json
 import shlex
 from subprocess import PIPE, Popen, TimeoutExpired
 from subprocess import run as subprocess_run
 
+import jwebmail.model.jwebmail_pb2 as pb2
+
 
 class QMAuthError(Exception):
     def __init__(self, msg, rc, response=None):
@@ -30,6 +31,7 @@ class QMailAuthuser:
                 input=f"{self._username}\0{self._password}\0\0".encode(),
                 shell=True,
                 timeout=2,
+                check=False,
             )
             if completed_proc.returncode == 0:
                 return True
@@ -41,87 +43,207 @@ class QMailAuthuser:
             return False
 
     def read_headers_for(self, folder, start, end, sort):
-        return self.build_and_run("list", [folder, start, end, sort])
+        req = pb2.ListReq(folder=folder, start=start, end=end, sort=sort)
+        resp = self.build_and_run("list", req.SerializeToString())
+        r = pb2.ListResp()
+        r.ParseFromString(resp)
+        return [
+            {
+                "message_handle": lmh.mid,
+                "byte_size": lmh.byte_size,
+                "unread": lmh.unread,
+                "date_received": lmh.rec_date,
+                "head": self._mail_header(lmh.header),
+            }
+            for lmh in r.mail_heads
+        ]
 
     def count(self, folder):
-        return self.build_and_run("count", [folder])
-
+        req = pb2.StatsReq(folder=folder)
+        resp = self.build_and_run("count", req.SerializeToString())
+        r = pb2.StatsResp()
+        r.ParseFromString(resp)
+        return {
+            "byte_size": r.byte_size,
+            "total_mails": r.mail_count,
+            "unread_mails": r.unread_count,
+        }
+
+    #
     def show(self, folder, msgid):
-        return self.build_and_run("read", [folder, msgid])
+        req = pb2.ShowReq(folder=folder, mid=msgid)
+        resp = self.build_and_run("read", req.SerializeToString())
+        r = pb2.ShowResp()
+        r.ParseFromString(resp)
+        return {
+            "head": self._mail_header(r.mail.head),
+            "body": self._mail_body(r.mail.body),
+        }
 
     def raw(self, folder, mid, path):
-        return self.build_and_run("raw", [folder, mid, path])
+        req = pb2.RawReq(folder=folder, mid=mid, path=path)
+        resp = self.build_and_run("raw", req.SerializeToString())
+        r = pb2.RawResp()
+        r.ParseFromString(resp)
+        return {"head": self._mime_header(r.header), "body": r.body}
 
+    #
     def search(self, pattern, folder):
-        return self.build_and_run("search", [pattern, folder])
+        req = pb2.SearchReq(folder=folder, pattern=pattern)
+        resp = self.build_and_run("search", req.SerializeToString())
+        r = pb2.SearchResp()
+        r.ParseFromString(resp)
+        return r
 
     def folders(self):
-        res = self.build_and_run("folders")
+        resp = self.build_and_run("folders", pb2.FoldersReq().SerializeToString())
+        result = pb2.FoldersResp()
+        result.ParseFromString(resp)
+        res = result.folders
         if isinstance(res, list):
             return [""] + res
         return res
 
     def move(self, mid, from_f, to_f):
-        _resp = self.build_and_run("move", [mid, from_f, to_f])
+        req = pb2.MoveReq(mid=mid, from_f=from_f, to_f=to_f)
+        resp = self.build_and_run("move", req.SerializeToString())
+        r = pb2.MoveResp()
+        r.ParseFromString(resp)
         return True
 
     def remove(self, folder, msgid):
-        _resp = self.build_and_run("remove", [folder, msgid])
+        req = pb2.RemoveReq(folder=folder, mid=msgid)
+        resp = self.build_and_run("remove", req.SerializeToString())
+        r = pb2.RemoveResp()
+        r.ParseFromString(resp)
         return True
 
-    def _build_arg(self, user_mail_addr, mode, args):
+    @staticmethod
+    def _address(addr):
+        if not addr:
+            return None
+        a = {"address": addr.address}
+        if addr.HasField("name"):
+            a["display_name"] = addr.name
+        return a
+
+    @classmethod
+    def _mail_header(cls, mail_head):
+        h = mail_head
+        return {
+            "date": h.send_date,
+            "sender": cls._address(h.sender),
+            "cc": [cls._address(x) for x in h.cc if x],
+            "bcc": [cls._address(x) for x in h.bcc if x],
+            "from": [cls._address(x) for x in h.written_from if x],
+            "reply_to": [cls._address(x) for x in h.reply_to if x],
+            "to": [cls._address(x) for x in h.send_to if x],
+            "subject": h.subject,
+            "comments": h.comments,
+            "keywords": h.keywords,
+            "mime": cls._mime_header(h.mime),
+        }
+
+    @staticmethod
+    def _mime_header(mime_head):
+        mh = {
+            "content_maintype": mime_head.maintype,
+            "content_subtype": mime_head.subtype,
+        }
+        if (
+            mime_head.contentdispo
+            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_NONE
+        ):
+            mh["content_disposition"] = None
+        elif (
+            mime_head.contentdispo
+            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_INLINE
+        ):
+            mh["content_disposition"] = "inline"
+        elif (
+            mime_head.contentdispo
+            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
+        ):
+            mh["content_disposition"] = "attachment"
+
+        if mime_head.HasField("file_name"):
+            mh["filename"] = mime_head.file_name
+
+        return mh
+
+    @classmethod
+    def _mail_body(cls, body):
+        if body.WhichOneof("Body") == "discrete":
+            return body.discrete
+        elif body.WhichOneof("Body") == "multipart":
+            res = {"parts": []}
+            for mp in body.multipart.parts:
+                r = {"head": cls._mime_header(mp.mime_header)}
+
+                if (
+                    mp.mime_header.contentdispo
+                    != pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
+                ):
+                    r["body"] = cls._mail_body(mp.body)
+
+                res["parts"].append(r)
+            if body.multipart.HasField("preamble"):
+                res["preamble"] = body.multipart.preamble
+            if body.multipart.HasField("epilogue"):
+                res["epilogue"] = body.multipart.epilogue
+            return res
+        elif body.WhichOneof("Body") == "mail":
+            return {
+                "head": cls._mail_header(body.mail.head),
+                "body": cls._mail_body(body.mail.body),
+            }
+        else:
+            assert False
+
+    def _build_arg(self, user_mail_addr, mode):
         idx = user_mail_addr.find("@")
         user_name = user_mail_addr[:idx]
 
         return (
             " ".join(
                 shlex.quote(str(x))
-                for x in [
+                for x in (
                     self._authenticator,
                     self._prog,
                     self._mailbox_path,
                     self._virtual_user,
                     user_name,
                     mode,
-                    *args,
-                ]
+                )
             )
             + " 3<&0"
         )
 
-    def _read_qmauth(self, prog_and_args):
-        popen = Popen(prog_and_args, stdin=PIPE, stdout=PIPE, shell=True)
+    def _read_qmauth(self, cmd, args):
+        popen = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=True, bufsize=0)
 
-        try:
-            inp, _ = popen.communicate(
-                f"{self._username}\0{self._password}\0\0".encode(), timeout=30
-            )
-            popen.wait(1)
-        except TimeoutExpired:
-            popen.terminate()
+        popen.stdin.write(f"{self._username}\0{self._password}\0\0".encode())
+        popen.stdin.flush()
+        r = popen.stdout.read(10)
+        if popen.poll():
+            raise QMAuthError("qmail-authuser unexpectedly exited", popen.returncode, r)
+        assert r == b"OPEN\n"
+        popen.stdin.write(args)
+        popen.stdin.close()
+        inp = popen.stdout.readall()
 
         if popen.poll() is None:
             popen.kill()
             popen.poll()
 
         rc = popen.returncode
-        if rc in [0, 3]:
-            try:
-                a, b, c = inp.partition(b"\n")
-                d = json.loads(a)
-                if b:
-                    resp = dict(head=d, body=c)
-                else:
-                    resp = d
-            except json.JSONDecodeError:
-                raise QMAuthError("error decoding response", rc, inp)
-            if rc == 3:
-                raise QMAuthError("error reported by extractor", rc, resp)
+        if rc == 0:
+            return inp
+        elif rc == 3:
+            raise QMAuthError("error reported by extractor", 3, inp)
         else:
             raise QMAuthError("got unsuccessful return code by qmail-authuser", rc, inp)
 
-        return resp
-
-    def build_and_run(self, mode, args=()):
-        prog_and_args = self._build_arg(self._username, mode, args)
-        return self._read_qmauth(prog_and_args)
+    def build_and_run(self, mode, args):
+        cmd = self._build_arg(self._username, mode)
+        return self._read_qmauth(cmd, args)