switch from protobuf based protocol to varlink

3 files changed, 227 insertions, 345 deletions

diff --git a/src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink b/src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink
new file mode 100644
index 0000000..64a4073
--- /dev/null
+++ b/src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink
@@ -0,0 +1,85 @@
+interface de.jmhoffmann.jwebmail.mail-storage
+
+
+type MIMEHeader (
+  mime_type:     (main_type: string, sub_type: string),
+  content_dispo: (none, inline, attachment),
+  file_name:     ?string
+)
+
+type MailAddr (
+  name: ?string,
+  address: string
+)
+
+# send_date is of ISO 8601 date-time format
+type MailHeader (
+  send_date:    string,
+  written_from: []MailAddr,
+  sender:       ?MailAddr,
+  reply_to:     []MailAddr,
+  send_to:      []MailAddr,
+  cc:           []MailAddr,
+  bcc:          []MailAddr,
+  subject:      string,
+  comments:     []string,
+  keywords:     []string,
+  mime:         MIMEHeader
+)
+
+type ListMailHeader (
+  byte_size: int,
+  unread:    bool,
+  rec_date:  string,
+  mid:       string,
+  header:    MailHeader
+)
+
+type Multipart (
+  preamble: ?string,
+  parts:    []MIMEPart,
+  epilogue: ?string
+)
+
+# exactly one of these fileds must be present
+type MailBody (
+  discrete:  ?string,
+  multipart: ?Multipart,
+  mail:      ?Mail
+)
+
+type Mail (
+  head: MailHeader,
+  body: MailBody
+)
+
+type MIMEPart (
+  mime_header: MIMEHeader,
+  body:        MailBody
+)
+
+type Sort (
+  direction: (asc, desc),
+  parameter: (date, size, sender)
+)
+
+
+method Init(unix_user: string, mailbox_path: string) -> ()
+method List(folder: string, start: int, end: int, sort: Sort) -> (mail_heads: []ListMailHeader)
+method Stats(folder: string) -> (mail_count: int, unread_count: int, byte_size: int)
+method Show(folder: string, mid: string) -> (mail: Mail)
+method Raw(folder: string, mid: string, path: ?string) -> (header: MIMEHeader, body: string)  # body is base64 encoded
+method Search(folder: string, pattern: string) -> (found: []ListMailHeader)
+method Folders() -> (folders: []string)
+method Move(mid: string, from_folder: string, to_folder: string) -> ()
+method Remove(folder: string, mid: string) -> ()
+method AddFolder(name: string) -> (status: (created, skiped))
+
+
+error NotIninitialized()
+error InvalidFolder(folder: string)
+error InvalidMID(folder: string, mid: string)
+error InvalidPathInMail(folder: string, mid: string, path: string)
+error InvalidSearchPattern(pattern: string)
+error InvalidUser(unix_user: string)
+error InvalidMailbox(path: string, not_a_mailbox: bool, user_mismatch: bool)
diff --git a/src/jwebmail/model/jwebmail.proto b/src/jwebmail/model/jwebmail.proto
deleted file mode 100644
index e4cba3b..0000000
--- a/src/jwebmail/model/jwebmail.proto
+++ /dev/null
@@ -1,165 +0,0 @@
-syntax = "proto3";
-
-package jwebmail;
-
-message MIMEHeader {
-
-  enum ContentDisposition {
-    CONTENT_DISPOSITION_NONE       = 0;
-    CONTENT_DISPOSITION_INLINE     = 1;
-    CONTENT_DISPOSITION_ATTACHMENT = 2;
-  }
-
-  string             maintype     = 1;
-  string             subtype      = 2;
-  ContentDisposition contentdispo = 3;
-  optional string    file_name    = 4;
-}
-
-message MailHeader {
-
-  message MailAddr {
-    optional string name    = 1;
-    string          address = 2;
-  }
-
-  string            send_date    =  1;
-  repeated MailAddr written_from =  2;
-  optional MailAddr sender       =  3;
-  repeated MailAddr reply_to     =  4;
-  repeated MailAddr send_to      =  5;
-  repeated MailAddr cc           =  6;
-  repeated MailAddr bcc          =  7;
-  string            subject      =  8;
-  repeated string   comments     =  9;
-  repeated string   keywords     = 10;
-  MIMEHeader        mime         = 11;
-}
-
-message ListMailHeader {
-  uint64     byte_size  = 1;
-  bool       unread     = 2;
-  string     rec_date   = 3;
-  string     mid        = 4;
-  MailHeader header     = 5;
-}
-
-message MailBody {
-  message Multipart {
-    optional string   preamble = 1;
-    repeated MIMEPart parts    = 2;
-    optional string   epilogue = 3;
-  }
-
-  oneof Body {
-    string    discrete  = 1;
-    Multipart multipart = 2;
-    Mail      mail      = 3;
-  }
-}
-
-message Mail {
-  MailHeader head = 1;
-  MailBody   body = 2;
-}
-
-message MIMEPart {
-  MIMEHeader mime_header = 1;
-  MailBody   body        = 2;
-}
-
-// Request-Response pairs
-
-message ListReq {
-  string folder = 1;
-  int32  start  = 2;
-  int32  end    = 3;
-  string sort   = 4;
-}
-
-message ListResp {
-  repeated ListMailHeader mail_heads = 1;
-}
-
-message StatsReq {
-  string folder = 1;
-}
-
-message StatsResp {
-  uint32 mail_count   = 1;
-  uint32 unread_count = 2;
-  uint64 byte_size    = 3;
-}
-
-message ShowReq {
-  string folder = 1;
-  string mid    = 2;
-}
-
-message ShowResp {
-  Mail mail = 1;
-}
-
-message RawReq {
-  string          folder = 1;
-  string          mid    = 2;
-  optional string path   = 3;
-}
-
-message RawResp {
-  MIMEHeader header = 1;
-  bytes      body   = 2;
-}
-
-message SearchReq {
-  string folder  = 1;
-  string pattern = 2;
-}
-
-message SearchResp {
-  repeated ListMailHeader found = 1;
-}
-
-message FoldersReq {
-}
-
-message FoldersResp {
-  repeated string folders = 1;
-}
-
-message MoveReq {
-  string mid    = 1;
-  string from_f = 2;
-  string to_f   = 3;
-}
-
-message MoveResp {
-}
-
-message RemoveReq {
-  string folder = 1;
-  string mid    = 2;
-}
-
-message RemoveResp {
-}
-
-message AddFolderReq {
-  string name = 1;
-}
-
-message AddFolderResp {
-  int32 status = 1;
-}
-
-service MailService {
-  rpc List(ListReq)       returns (ListResp);
-  rpc Stats(StatsReq)     returns (StatsResp);
-  rpc Show(ShowReq)       returns (ShowResp);
-  rpc Raw(RawReq)         returns (RawResp);
-  rpc Search(SearchReq)   returns (SearchResp);
-  rpc Folders(FoldersReq) returns (FoldersResp);
-  rpc Move(MoveReq)       returns (MoveResp);
-  rpc Remove(RemoveReq)   returns (RemoveResp);
-  rpc AddFolder(AddFolderReq) returns (AddFolderResp);
-}
diff --git a/src/jwebmail/model/read_mails.py b/src/jwebmail/model/read_mails.py
index 5c63bdd..8a19224 100644
--- a/src/jwebmail/model/read_mails.py
+++ b/src/jwebmail/model/read_mails.py
@@ -1,16 +1,14 @@
 import os
-from subprocess import PIPE, Popen, TimeoutExpired
-from subprocess import run as subprocess_run
+from base64 import b64decode
+from socket import socketpair
 
-import jwebmail.model.jwebmail_pb2 as pb2
+import varlink
 
 
 class QMAuthError(Exception):
-    def __init__(self, msg, rc, response=None):
-        super().__init__(msg, rc, response)
-        self.msg = msg
+    def __init__(self, rc):
+        super().__init__(rc)
         self.rc = rc
-        self.response = response
 
 
 class QMailAuthuser:
@@ -24,235 +22,199 @@ class QMailAuthuser:
         self._virtual_user = virtual_user
         self._authenticator = authenticator
 
-    def verify_user(self):
-        try:
-            completed_proc = subprocess_run(
-                f"{self._authenticator} true 3<&0",
-                input=f"{self._username}\0{self._password}\0\0".encode(),
-                shell=True,
-                timeout=2,
-                check=False,
-            )
-            if completed_proc.returncode == 0:
-                return True
-            elif completed_proc.returncode == 1:
-                return False
-            else:
-                raise QMAuthError("authentication error", completed_proc.returncode)
-        except TimeoutExpired:
-            return False
+        self._pid = None
+        self._socket = None
+        self._client = None
+        self._connection = None
 
     def read_headers_for(self, folder, start, end, sort):
-        req = pb2.ListReq(folder=folder, start=start, end=end, sort=sort)
-        resp = self.build_and_run("list", req.SerializeToString())
-        r = pb2.ListResp()
-        r.ParseFromString(resp)
+        sort_val = dict()
+        if sort[0] == "!":
+            sort = sort[1:]
+            sort_val["direction"] = "desc"
+        else:
+            sort_val["direction"] = "asc"
+
+        match sort:
+            case "date" | "subject" | "size":
+                sort_val["parameter"] = sort
+            case _:
+                raise ValueError(f"invalid sort parameter {sort!r}")
+
+        req = self._connection.List(folder=folder, start=start, end=end, sort=sort_val)
         return [
             {
-                "message_handle": lmh.mid,
-                "byte_size": lmh.byte_size,
-                "unread": lmh.unread,
-                "date_received": lmh.rec_date,
-                "head": self._mail_header(lmh.header),
+                "message_handle": lmh["mid"],
+                "byte_size": lmh["byte_size"],
+                "unread": lmh["unread"],
+                "date_received": lmh["rec_date"],
+                "head": self._mail_header(lmh["header"]),
             }
-            for lmh in r.mail_heads
+            for lmh in req["mail_heads"]
         ]
 
     def count(self, folder):
-        req = pb2.StatsReq(folder=folder)
-        resp = self.build_and_run("count", req.SerializeToString())
-        r = pb2.StatsResp()
-        r.ParseFromString(resp)
+        resp = self._connection.Stats(folder=folder)
         return {
-            "byte_size": r.byte_size,
-            "total_mails": r.mail_count,
-            "unread_mails": r.unread_count,
+            "byte_size": resp["byte_size"],
+            "total_mails": resp["mail_count"],
+            "unread_mails": resp["unread_count"],
         }
 
     def show(self, folder, msgid):
-        req = pb2.ShowReq(folder=folder, mid=msgid)
-        resp = self.build_and_run("read", req.SerializeToString())
-        r = pb2.ShowResp()
-        r.ParseFromString(resp)
+        resp = self._connection.Show(folder=folder, mid=msgid)
         return {
-            "head": self._mail_header(r.mail.head),
-            "body": self._mail_body(r.mail.body),
+            "head": self._mail_header(resp["mail"]["head"]),
+            "body": self._mail_body(resp["mail"]["body"]),
         }
 
     def raw(self, folder, mid, path):
-        req = pb2.RawReq(folder=folder, mid=mid, path=path)
-        resp = self.build_and_run("raw", req.SerializeToString())
-        r = pb2.RawResp()
-        r.ParseFromString(resp)
-        return {"head": self._mime_header(r.header), "body": r.body}
+        resp = self._connection.Raw(folder=folder, mid=mid, path=path)
+        return {
+            "head": self._mime_header(resp["header"]),
+            "body": b64decode(resp["body"]),
+        }
 
-    #
     def search(self, pattern, folder):
-        req = pb2.SearchReq(folder=folder, pattern=pattern)
-        resp = self.build_and_run("search", req.SerializeToString())
-        r = pb2.SearchResp()
-        r.ParseFromString(resp)
-        return r
+        resp = self._connection.Search(folder=folder, pattern=pattern)
+        return resp
 
     def folders(self):
-        resp = self.build_and_run("folders", pb2.FoldersReq().SerializeToString())
-        result = pb2.FoldersResp()
-        result.ParseFromString(resp)
-        return list(result.folders) + [""]
+        resp = self._connection.Folders()
+        return list(resp["folders"]) + [""]
 
     def move(self, mid, from_f, to_f):
-        req = pb2.MoveReq(mid=mid, from_f=from_f, to_f=to_f)
-        resp = self.build_and_run("move", req.SerializeToString())
-        r = pb2.MoveResp()
-        r.ParseFromString(resp)
+        self._connection.Move(mid=mid, from_folder=from_f, to_folder=to_f)
         return True
 
     def remove(self, folder, msgid):
-        req = pb2.RemoveReq(folder=folder, mid=msgid)
-        resp = self.build_and_run("remove", req.SerializeToString())
-        r = pb2.RemoveResp()
-        r.ParseFromString(resp)
+        self._connection.Remove(folder=folder, mid=msgid)
         return True
 
     def add_folder(self, name):
-        req = pb2.AddFolderReq(name=name)
-        resp = self.build_and_run("add-folder", req.SerializeToString())
-        r = pb2.AddFolderResp()
-        r.ParseFromString(resp)
-        return r.status
+        resp = self._connection.AddFolder(name=name)
+        return resp["status"]
 
     @staticmethod
     def _address(addr):
         if not addr:
             return None
-        a = {"address": addr.address}
-        if addr.HasField("name"):
-            a["display_name"] = addr.name
+        a = {"address": addr["address"]}
+        if addr.get("name"):
+            a["display_name"] = addr["name"]
         return a
 
-    @classmethod
-    def _mail_header(cls, mail_head):
+    def _mail_header(self, mail_head):
         h = mail_head
         return {
-            "date": h.send_date,
-            "sender": cls._address(h.sender),
-            "cc": [cls._address(x) for x in h.cc if x],
-            "bcc": [cls._address(x) for x in h.bcc if x],
-            "from": [cls._address(x) for x in h.written_from if x],
-            "reply_to": [cls._address(x) for x in h.reply_to if x],
-            "to": [cls._address(x) for x in h.send_to if x],
-            "subject": h.subject,
-            "comments": list(h.comments),
-            "keywords": list(h.keywords),
-            "mime": cls._mime_header(h.mime),
+            "date": h["send_date"],
+            "sender": self._address(h.get("sender")),
+            "cc": [self._address(x) for x in h["cc"] if x],
+            "bcc": [self._address(x) for x in h["bcc"] if x],
+            "from": [self._address(x) for x in h["written_from"] if x],
+            "reply_to": [self._address(x) for x in h["reply_to"] if x],
+            "to": [self._address(x) for x in h["send_to"] if x],
+            "subject": h["subject"],
+            "comments": list(h["comments"]),
+            "keywords": list(h["keywords"]),
+            "mime": self._mime_header(h["mime"]),
         }
 
     @staticmethod
     def _mime_header(mime_head):
         mh = {
-            "content_maintype": mime_head.maintype,
-            "content_subtype": mime_head.subtype,
+            "content_maintype": mime_head["mime_type"]["main_type"],
+            "content_subtype": mime_head["mime_type"]["sub_type"],
         }
-        if (
-            mime_head.contentdispo
-            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_NONE
-        ):
-            mh["content_disposition"] = None
-        elif (
-            mime_head.contentdispo
-            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_INLINE
-        ):
-            mh["content_disposition"] = "inline"
-        elif (
-            mime_head.contentdispo
-            == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
-        ):
-            mh["content_disposition"] = "attachment"
-
-        if mime_head.HasField("file_name"):
-            mh["filename"] = mime_head.file_name
+        match mime_head["content_dispo"]:
+            case "inline":
+                mh["content_disposition"] = "inline"
+            case "attachment":
+                mh["content_disposition"] = "attachment"
+            case "none":
+                mh["content_disposition"] = None
+            case invalid:
+                raise ValueError(f"invalid content disposition {invalid!r}")
+
+        if "file_name" in mime_head:
+            mh["filename"] = mime_head["file_name"]
 
         return mh
 
-    @classmethod
-    def _mail_body(cls, body):
-        if body.WhichOneof("Body") == "discrete":
-            return body.discrete
-        elif body.WhichOneof("Body") == "multipart":
+    def _mail_body(self, body):
+        assert len(body) == 1
+
+        if "discrete" in body:
+            return body["discrete"]
+        elif "multipart" in body:
             res = {"parts": []}
-            for mp in body.multipart.parts:
-                r = {"head": cls._mime_header(mp.mime_header)}
+            for mp in body["multipart"]["parts"]:
+                r = {"head": self._mime_header(mp["mime_header"])}
 
-                if (
-                    mp.mime_header.contentdispo
-                    != pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT
-                ):
-                    r["body"] = cls._mail_body(mp.body)
+                if mp["mime_header"]["content_dispo"] != "attachment":
+                    r["body"] = self._mail_body(mp["body"])
 
                 res["parts"].append(r)
-            if body.multipart.HasField("preamble"):
-                res["preamble"] = body.multipart.preamble
-            if body.multipart.HasField("epilogue"):
-                res["epilogue"] = body.multipart.epilogue
+            if pr := body["multipart"].get("preamble"):
+                res["preamble"] = pr
+            if ep := body["multipart"].get("epilogue"):
+                res["epilogue"] = ep
             return res
-        elif body.WhichOneof("Body") == "mail":
+        elif "mail" in body:
             return {
-                "head": cls._mail_header(body.mail.head),
-                "body": cls._mail_body(body.mail.body),
+                "head": self._mail_header(body["mail"]["head"]),
+                "body": self._mail_body(body["mail"]["body"]),
             }
         else:
             assert False
 
-    def _build_arg(self, user_mail_addr, mode, rp):
-        idx = user_mail_addr.find("@")
-        user_name = user_mail_addr[:idx]
-
-        cmdline = [
-            "moveto3.py",
-            "-a",
-            self._authenticator,
-            str(rp),
-            self._prog,
-            self._mailbox_path,
-            self._virtual_user,
-            user_name,
-            mode,
-        ]
-
-        return cmdline
-
-    def _read_qmauth(self, cmd, args, rp, wp):
-
-        with Popen(cmd, stdin=PIPE, stdout=PIPE, pass_fds=[rp], bufsize=0) as popen:
-            os.close(rp)
-            os.write(wp, f"{self._username}\0{self._password}\0\0".encode())
+    def open(self):
+        (rp, wp) = os.pipe()
+        (sp, sc) = socketpair()
+        cmdline = [self._authenticator, self._prog]
+        if (pid := os.fork()) == 0:
+            # child
             os.close(wp)
-            r = popen.stdout.read(10)
-            if popen.poll():
-                raise QMAuthError(
-                    "qmail-authuser unexpectedly exited", popen.returncode, r
-                )
-            assert r == b"OPEN\n"
-            popen.stdin.write(args)
-            popen.stdin.close()
-            inp = popen.stdout.readall()
-
-            try:
-                popen.wait(timeout=2)
-            except TimeoutExpired:
-                popen.kill()
-                popen.poll()
-
-            rc = popen.returncode
+            sp.close()
+            os.dup2(rp, 3)
+            os.dup2(sc.fileno(), 4)
+            os.environ["LISTEN_FDS"] = "2"
+            os.environ["LISTEN_FDNAMES"] = "auth:varlink"
+            os.environ["LISTEN_PID"] = str(os.getpid())
+            os.execvp(self._authenticator, cmdline)
+            assert False
+        sc.close()
+        os.close(rp)
+        os.write(wp, f"{self._username}\0{self._password}\0\0".encode())
+        os.close(wp)
 
-        if rc == 0:
-            return inp
-        elif rc == 3:
-            raise QMAuthError("error reported by extractor", 3, inp)
-        else:
-            raise QMAuthError("got unsuccessful return code by qmail-authuser", rc, inp)
+        self._pid = pid
+        self._socket = sp
+        self._client = varlink.Client()
 
-    def build_and_run(self, mode, args):
-        (rp, wp) = os.pipe()
-        cmd = self._build_arg(self._username, mode, rp)
-        return self._read_qmauth(cmd, args, rp, wp)
+        try:
+            self._connection = self._client.open(
+                "de.jmhoffmann.jwebmail.mail-storage", connection=sp
+            )
+        except ConnectionResetError:
+            (pid, status) = os.waitpid(pid, os.WNOHANG)
+            if pid != 0:
+                raise QMAuthError(os.waitstatus_to_exitcode(status))
+            else:
+                raise
+
+        user = self._username[: self._username.index("@")]
+        self._connection.Init(
+            unix_user=self._virtual_user,
+            mailbox_path=os.path.join(self._mailbox_path, user),
+        )
+        return self
+
+    def close(self):
+        self._connection.close()
+        self._socket.close()
+        (pid, status) = os.waitpid(self._pid, 0)
+        assert pid == self._pid
+        rc = os.waitstatus_to_exitcode(status)
+        if rc != 0:
+            raise QMAuthError(rc)