convert internal message format from json to protobuf

2 files changed, 318 insertions, 40 deletions

diff --git a/src/jwebmail/model/jwebmail.proto b/src/jwebmail/model/jwebmail.proto
new file mode 100644
index 0000000..bf1454a
--- /dev/null
+++ b/src/jwebmail/model/jwebmail.proto
@@ -0,0 +1,156 @@
+syntax = "proto3";
+
+package jwebmail;
+
+message MIMEHeader {
+
+  enum ContentDisposition {
+    CONTENT_DISPOSITION_NONE       = 0;
+    CONTENT_DISPOSITION_INLINE     = 1;
+    CONTENT_DISPOSITION_ATTACHMENT = 2;
+  }
+
+  string             maintype     = 1;
+  string             subtype      = 2;
+  ContentDisposition contentdispo = 3;
+  optional string    file_name    = 4;
+}
+
+message MailHeader {
+
+  message MailAddr {
+    optional string name    = 1;
+    string          address = 2;
+  }
+
+  string            send_date    =  1;
+  repeated MailAddr written_from =  2;
+  optional MailAddr sender       =  3;
+  repeated MailAddr reply_to     =  4;
+  repeated MailAddr send_to      =  5;
+  repeated MailAddr cc           =  6;
+  repeated MailAddr bcc          =  7;
+  string            subject      =  8;
+  repeated string   comments     =  9;
+  repeated string   keywords     = 10;
+  MIMEHeader        mime         = 11;
+}
+
+message ListMailHeader {
+  uint64     byte_size  = 1;
+  bool       unread     = 2;
+  string     rec_date   = 3;
+  string     mid        = 4;
+  MailHeader header     = 5;
+}
+
+message MailBody {
+  message Multipart {
+    optional string   preamble = 1;
+    repeated MIMEPart parts    = 2;
+    optional string   epilogue = 3;
+  }
+
+  oneof Body {
+    string    discrete  = 1;
+    Multipart multipart = 2;
+    Mail      mail      = 3;
+  }
+}
+
+message Mail {
+  MailHeader head = 1;
+  MailBody   body = 2;
+}
+
+message MIMEPart {
+  MIMEHeader mime_header = 1;
+  MailBody   body        = 2;
+}
+
+// Request-Response pairs
+
+message ListReq {
+  string folder = 1;
+  int32  start  = 2;
+  int32  end    = 3;
+  string sort   = 4;
+}
+
+message ListResp {
+  repeated ListMailHeader mail_heads = 1;
+}
+
+message StatsReq {
+  string folder = 1;
+}
+
+message StatsResp {
+  uint32 mail_count   = 1;
+  uint32 unread_count = 2;
+  uint64 byte_size    = 3;
+}
+
+message ShowReq {
+  string folder = 1;
+  string mid    = 2;
+}
+
+message ShowResp {
+  Mail mail = 1;
+}
+
+message RawReq {
+  string          folder = 1;
+  string          mid    = 2;
+  optional string path   = 3;
+}
+
+message RawResp {
+  MIMEHeader header = 1;
+  bytes      body   = 2;
+}
+
+message SearchReq {
+  string folder  = 1;
+  string pattern = 2;
+}
+
+message SearchResp {
+  repeated ListMailHeader found = 1;
+}
+
+message FoldersReq {
+}
+
+message FoldersResp {
+  repeated string folders = 1;
+}
+
+message MoveReq {
+  string mid    = 1;
+  string from_f = 2;
+  string to_f   = 3;
+}
+
+message MoveResp {
+}
+
+message RemoveReq {
+  string folder = 1;
+  string mid    = 2;
+}
+
+message RemoveResp {
+}
+
+service MailService {
+  rpc List(ListReq)       returns (ListResp);
+  rpc Stats(StatsReq)     returns (StatsResp);
+  rpc Show(ShowReq)       returns (ShowResp);
+  rpc Raw(RawReq)         returns (RawResp);
+  rpc Search(SearchReq)   returns (SearchResp);
+  rpc Folders(FoldersReq) returns (FoldersResp);
+  rpc Move(MoveReq)       returns (MoveResp);
+  rpc Remove(RemoveReq)   returns (RemoveResp);
+}
diff --git a/src/jwebmail/model/read_mails.py b/src/jwebmail/model/read_mails.py
index 45d5996..8ba3c67 100644
--- a/src/jwebmail/model/read_mails.py
+++ b/src/jwebmail/model/read_mails.py
@@ -1,8 +1,9 @@
-import json
 import shlex
 from subprocess import PIPE, Popen, TimeoutExpired
 from subprocess import run as subprocess_run
 
+import jwebmail.model.jwebmail_pb2 as pb2
+
 
 class QMAuthError(Exception):
     def __init__(self, msg, rc, response=None):
@@ -30,6 +31,7 @@ class QMailAuthuser:
                 input=f"{self._username}\0{self._password}\0\0".encode(),
                 shell=True,
                 timeout=2,
+                check=False,
             )
             if completed_proc.returncode == 0:
                 return True
@@ -41,87 +43,207 @@ class QMailAuthuser:
             return False
 
     def read_headers_for(self, folder, start, end, sort):
-        return self.build_and_run("list", [folder, start, end, sort])
+        req = pb2.ListReq(folder=folder, start=start, end=end, sort=sort)
+        resp = self.build_and_run("list", req.SerializeToString())
+        r = pb2.ListResp()
+        r.ParseFromString(resp)
+        return [
+            {
+                "message_handle": lmh.mid,
+                "byte_size": lmh.byte_size,
+                "unread": lmh.unread,
+                "date_received": lmh.rec_date,
+                "head": self._mail_header(lmh.header),
+            }
+            for lmh in r.mail_heads
+        ]
 
     def count(self, folder):
-        return self.build_and_run("count", [folder])
-
+        req = pb2.StatsReq(folder=folder)
+        resp = self.build_and_run("count", req.SerializeToString())
+        r = pb2.StatsResp()
+        r.ParseFromString(resp)
+        return {
+            "byte_size": r.byte_size,
+            "total_mails": r.mail_count,
+            "unread_mails": r.unread_count,
+        }
+
+    #
     def show(self, folder, msgid):
-        return self.build_and_run("read", [folder, msgid])
+        req = pb2.ShowReq(folder=folder, mid=msgid)
+        resp = self.build_and_run("read", req.SerializeToString())
+        r = pb2.ShowResp()
+        r.ParseFromString(resp)
+        return {
+            "head": self._mail_header(r.mail.head),
+            "body": self._mail_body(r.mail.body),
+        }
 
     def raw(self, folder, mid, path):
-        return self.build_and_run("raw", [folder, mid, path])
+        req = pb2.RawReq(folder=folder, mid=mid, path=path)
+        resp = self.build_and_run("raw", req.SerializeToString())
+        r = pb2.RawResp()
+        r.ParseFromString(resp)
+        return {"head": self._mime_header(r.header), "body": r.body}
 
+    #
     def search(self, pattern, folder):
-        return self.build_and_run("search", [pattern, folder])
+        req = pb2.SearchReq(folder=folder, pattern=pattern)
+        resp = self.build_and_run("search", req.SerializeToString())
+        r = pb2.SearchResp()
+        r.ParseFromString(resp)
+        return r
 
     def folders(self):
-        res = self.build_and_run("folders")
+        resp = self.build_and_run("folders", pb2.FoldersReq().SerializeToString())
+        result = pb2.FoldersResp()
+        result.ParseFromString(resp)
+        res = result.folders
         if isinstance(res, list):
             return [""] + res
         return res
 
     def move(self, mid, from_f, to_f):
-        _resp = self.build_and_run("move", [mid, from_f, to_f])
+        req = pb2.MoveReq(mid=mid, from_f=from_f, to_f=to_f)
+        resp = self.build_and_run("move", req.SerializeToString())
+        r = pb2.MoveResp()
+        r.ParseFromString(resp)
         return True
 
     def remove(self, folder, msgid):
-        _resp = self.build_and_run("remove", [folder, msgid])
+        req = pb2.RemoveReq(folder=folder, mid=msgid)
+        resp = self.build_and_run("remove", req.SerializeToString())
+        r = pb2.RemoveResp()
+        r.ParseFromString(resp)
         return True
 
-    def _build_arg(self, user_mail_addr, mode, args):
+    @staticmethod
+    def _address(addr):
+        if not addr:
+            return None
+        a = {"address": addr.address}
+        if addr.HasField("name"):
+            a["display_name"] = addr.name
+        return a
+
+    @classmethod
+    def _mail_header(cls, mail_head):
+        h = mail_head
+        return {
+            "date": h.send_date,
+            "sender": cls._address(h.sender),
+            "cc": [cls._address(x) for x in h.cc if x],
+            "bcc": [cls._address(x) for x in h.bcc if x],
+            "from": [cls._address(x) for x in h.written_from if x],
+            "reply_to": [cls._address(x) for x in h.reply_to if x],
+            "to": [cls._address(x) for x in h.send_to if x],
+            "subject": h.subject,
+            "comments": h.comments,
+            "keywords": h.keywords,
+            "mime": cls._mime_header(h.mime),
+        }
+
+    @staticmethod
+    def _mime_header(mime_head):
+        mh = {
+            "content_maintype": mime_head.maintype,
+            "content_subtype": mime_head.subtype,
+        }
+        if (
+            mime_head.contentdispo
+            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_NONE
+        ):
+            mh["content_disposition"] = None
+        elif (
+            mime_head.contentdispo
+            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_INLINE
+        ):
+            mh["content_disposition"] = "inline"
+        elif (
+            mime_head.contentdispo
+            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
+        ):
+            mh["content_disposition"] = "attachment"
+
+        if mime_head.HasField("file_name"):
+            mh["filename"] = mime_head.file_name
+
+        return mh
+
+    @classmethod
+    def _mail_body(cls, body):
+        if body.WhichOneof("Body") == "discrete":
+            return body.discrete
+        elif body.WhichOneof("Body") == "multipart":
+            res = {"parts": []}
+            for mp in body.multipart.parts:
+                r = {"head": cls._mime_header(mp.mime_header)}
+
+                if (
+                    mp.mime_header.contentdispo
+                    != pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
+                ):
+                    r["body"] = cls._mail_body(mp.body)
+
+                res["parts"].append(r)
+            if body.multipart.HasField("preamble"):
+                res["preamble"] = body.multipart.preamble
+            if body.multipart.HasField("epilogue"):
+                res["epilogue"] = body.multipart.epilogue
+            return res
+        elif body.WhichOneof("Body") == "mail":
+            return {
+                "head": cls._mail_header(body.mail.head),
+                "body": cls._mail_body(body.mail.body),
+            }
+        else:
+            assert False
+
+    def _build_arg(self, user_mail_addr, mode):
         idx = user_mail_addr.find("@")
         user_name = user_mail_addr[:idx]
 
         return (
             " ".join(
                 shlex.quote(str(x))
-                for x in [
+                for x in (
                     self._authenticator,
                     self._prog,
                     self._mailbox_path,
                     self._virtual_user,
                     user_name,
                     mode,
-                    *args,
-                ]
+                )
             )
             + " 3<&0"
         )
 
-    def _read_qmauth(self, prog_and_args):
-        popen = Popen(prog_and_args, stdin=PIPE, stdout=PIPE, shell=True)
+    def _read_qmauth(self, cmd, args):
+        popen = Popen(cmd, stdin=PIPE, stdout=PIPE, shell=True, bufsize=0)
 
-        try:
-            inp, _ = popen.communicate(
-                f"{self._username}\0{self._password}\0\0".encode(), timeout=30
-            )
-            popen.wait(1)
-        except TimeoutExpired:
-            popen.terminate()
+        popen.stdin.write(f"{self._username}\0{self._password}\0\0".encode())
+        popen.stdin.flush()
+        r = popen.stdout.read(10)
+        if popen.poll():
+            raise QMAuthError("qmail-authuser unexpectedly exited", popen.returncode, r)
+        assert r == b"OPEN\n"
+        popen.stdin.write(args)
+        popen.stdin.close()
+        inp = popen.stdout.readall()
 
         if popen.poll() is None:
             popen.kill()
             popen.poll()
 
         rc = popen.returncode
-        if rc in [0, 3]:
-            try:
-                a, b, c = inp.partition(b"\n")
-                d = json.loads(a)
-                if b:
-                    resp = dict(head=d, body=c)
-                else:
-                    resp = d
-            except json.JSONDecodeError:
-                raise QMAuthError("error decoding response", rc, inp)
-            if rc == 3:
-                raise QMAuthError("error reported by extractor", rc, resp)
+        if rc == 0:
+            return inp
+        elif rc == 3:
+            raise QMAuthError("error reported by extractor", 3, inp)
         else:
             raise QMAuthError("got unsuccessful return code by qmail-authuser", rc, inp)
 
-        return resp
-
-    def build_and_run(self, mode, args=()):
-        prog_and_args = self._build_arg(self._username, mode, args)
-        return self._read_qmauth(prog_and_args)
+    def build_and_run(self, mode, args):
+        cmd = self._build_arg(self._username, mode)
+        return self._read_qmauth(cmd, args)