From 5324a38f8fbd41391741317f7e7ab2d69ec30623 Mon Sep 17 00:00:00 2001 From: "Jannis M. Hoffmann" Date: Tue, 19 Nov 2024 23:15:55 +0100 Subject: switch from protobuf based protocol to varlink --- src/jwebmail/__init__.py | 14 +- .../de.jmhoffmann.jwebmail.mail-storage.varlink | 85 ++++++ src/jwebmail/model/jwebmail.proto | 165 ----------- src/jwebmail/model/read_mails.py | 322 +++++++++------------ src/jwebmail/read_mails.py | 15 +- src/jwebmail/render_mail.py | 4 +- 6 files changed, 251 insertions(+), 354 deletions(-) create mode 100644 src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink delete mode 100644 src/jwebmail/model/jwebmail.proto (limited to 'src/jwebmail') diff --git a/src/jwebmail/__init__.py b/src/jwebmail/__init__.py index 67018ce..f48db48 100644 --- a/src/jwebmail/__init__.py +++ b/src/jwebmail/__init__.py @@ -4,7 +4,7 @@ from os import environ from shutil import which from babel import parse_locale -from flask import Flask, abort, g, redirect, url_for +from flask import Flask, abort, g, redirect, request_finished, url_for from flask_babel import Babel, get_locale from flask_login import LoginManager, login_required from flask_wtf.csrf import CSRFProtect @@ -36,7 +36,7 @@ else: toml_read_file = dict(load=toml_load, text=True) -__version__ = "2.6.0.dev5" +__version__ = "2.7.0.dev0" csrf = CSRFProtect() @@ -63,7 +63,9 @@ def create_app(): app.config.from_file(environ["JWEBMAIL_CONFIG"], **toml_read_file) if app.config["JWEBMAIL"].get("PROXY"): - app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1) + app.wsgi_app = ProxyFix( + app.wsgi_app, x_for=True, x_proto=True, x_host=True, x_prefix=True + ) validate_config(app) @@ -120,6 +122,12 @@ def create_app(): except ValueError: abort(404) + @request_finished.connect_via(app) + def close_qma(_app, **_): + if "read_mails" in g: + g.read_mails.close() + g.read_mails = None + return app diff --git a/src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink b/src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink new file mode 100644 index 0000000..64a4073 --- /dev/null +++ b/src/jwebmail/model/de.jmhoffmann.jwebmail.mail-storage.varlink @@ -0,0 +1,85 @@ +interface de.jmhoffmann.jwebmail.mail-storage + + +type MIMEHeader ( + mime_type: (main_type: string, sub_type: string), + content_dispo: (none, inline, attachment), + file_name: ?string +) + +type MailAddr ( + name: ?string, + address: string +) + +# send_date is of ISO 8601 date-time format +type MailHeader ( + send_date: string, + written_from: []MailAddr, + sender: ?MailAddr, + reply_to: []MailAddr, + send_to: []MailAddr, + cc: []MailAddr, + bcc: []MailAddr, + subject: string, + comments: []string, + keywords: []string, + mime: MIMEHeader +) + +type ListMailHeader ( + byte_size: int, + unread: bool, + rec_date: string, + mid: string, + header: MailHeader +) + +type Multipart ( + preamble: ?string, + parts: []MIMEPart, + epilogue: ?string +) + +# exactly one of these fileds must be present +type MailBody ( + discrete: ?string, + multipart: ?Multipart, + mail: ?Mail +) + +type Mail ( + head: MailHeader, + body: MailBody +) + +type MIMEPart ( + mime_header: MIMEHeader, + body: MailBody +) + +type Sort ( + direction: (asc, desc), + parameter: (date, size, sender) +) + + +method Init(unix_user: string, mailbox_path: string) -> () +method List(folder: string, start: int, end: int, sort: Sort) -> (mail_heads: []ListMailHeader) +method Stats(folder: string) -> (mail_count: int, unread_count: int, byte_size: int) +method Show(folder: string, mid: string) -> (mail: Mail) +method Raw(folder: string, mid: string, path: ?string) -> (header: MIMEHeader, body: string) # body is base64 encoded +method Search(folder: string, pattern: string) -> (found: []ListMailHeader) +method Folders() -> (folders: []string) +method Move(mid: string, from_folder: string, to_folder: string) -> () +method Remove(folder: string, mid: string) -> () +method AddFolder(name: string) -> (status: (created, skiped)) + + +error NotIninitialized() +error InvalidFolder(folder: string) +error InvalidMID(folder: string, mid: string) +error InvalidPathInMail(folder: string, mid: string, path: string) +error InvalidSearchPattern(pattern: string) +error InvalidUser(unix_user: string) +error InvalidMailbox(path: string, not_a_mailbox: bool, user_mismatch: bool) diff --git a/src/jwebmail/model/jwebmail.proto b/src/jwebmail/model/jwebmail.proto deleted file mode 100644 index e4cba3b..0000000 --- a/src/jwebmail/model/jwebmail.proto +++ /dev/null @@ -1,165 +0,0 @@ -syntax = "proto3"; - -package jwebmail; - -message MIMEHeader { - - enum ContentDisposition { - CONTENT_DISPOSITION_NONE = 0; - CONTENT_DISPOSITION_INLINE = 1; - CONTENT_DISPOSITION_ATTACHMENT = 2; - } - - string maintype = 1; - string subtype = 2; - ContentDisposition contentdispo = 3; - optional string file_name = 4; -} - -message MailHeader { - - message MailAddr { - optional string name = 1; - string address = 2; - } - - string send_date = 1; - repeated MailAddr written_from = 2; - optional MailAddr sender = 3; - repeated MailAddr reply_to = 4; - repeated MailAddr send_to = 5; - repeated MailAddr cc = 6; - repeated MailAddr bcc = 7; - string subject = 8; - repeated string comments = 9; - repeated string keywords = 10; - MIMEHeader mime = 11; -} - -message ListMailHeader { - uint64 byte_size = 1; - bool unread = 2; - string rec_date = 3; - string mid = 4; - MailHeader header = 5; -} - -message MailBody { - message Multipart { - optional string preamble = 1; - repeated MIMEPart parts = 2; - optional string epilogue = 3; - } - - oneof Body { - string discrete = 1; - Multipart multipart = 2; - Mail mail = 3; - } -} - -message Mail { - MailHeader head = 1; - MailBody body = 2; -} - -message MIMEPart { - MIMEHeader mime_header = 1; - MailBody body = 2; -} - -// Request-Response pairs - -message ListReq { - string folder = 1; - int32 start = 2; - int32 end = 3; - string sort = 4; -} - -message ListResp { - repeated ListMailHeader mail_heads = 1; -} - -message StatsReq { - string folder = 1; -} - -message StatsResp { - uint32 mail_count = 1; - uint32 unread_count = 2; - uint64 byte_size = 3; -} - -message ShowReq { - string folder = 1; - string mid = 2; -} - -message ShowResp { - Mail mail = 1; -} - -message RawReq { - string folder = 1; - string mid = 2; - optional string path = 3; -} - -message RawResp { - MIMEHeader header = 1; - bytes body = 2; -} - -message SearchReq { - string folder = 1; - string pattern = 2; -} - -message SearchResp { - repeated ListMailHeader found = 1; -} - -message FoldersReq { -} - -message FoldersResp { - repeated string folders = 1; -} - -message MoveReq { - string mid = 1; - string from_f = 2; - string to_f = 3; -} - -message MoveResp { -} - -message RemoveReq { - string folder = 1; - string mid = 2; -} - -message RemoveResp { -} - -message AddFolderReq { - string name = 1; -} - -message AddFolderResp { - int32 status = 1; -} - -service MailService { - rpc List(ListReq) returns (ListResp); - rpc Stats(StatsReq) returns (StatsResp); - rpc Show(ShowReq) returns (ShowResp); - rpc Raw(RawReq) returns (RawResp); - rpc Search(SearchReq) returns (SearchResp); - rpc Folders(FoldersReq) returns (FoldersResp); - rpc Move(MoveReq) returns (MoveResp); - rpc Remove(RemoveReq) returns (RemoveResp); - rpc AddFolder(AddFolderReq) returns (AddFolderResp); -} diff --git a/src/jwebmail/model/read_mails.py b/src/jwebmail/model/read_mails.py index 5c63bdd..8a19224 100644 --- a/src/jwebmail/model/read_mails.py +++ b/src/jwebmail/model/read_mails.py @@ -1,16 +1,14 @@ import os -from subprocess import PIPE, Popen, TimeoutExpired -from subprocess import run as subprocess_run +from base64 import b64decode +from socket import socketpair -import jwebmail.model.jwebmail_pb2 as pb2 +import varlink class QMAuthError(Exception): - def __init__(self, msg, rc, response=None): - super().__init__(msg, rc, response) - self.msg = msg + def __init__(self, rc): + super().__init__(rc) self.rc = rc - self.response = response class QMailAuthuser: @@ -24,235 +22,199 @@ class QMailAuthuser: self._virtual_user = virtual_user self._authenticator = authenticator - def verify_user(self): - try: - completed_proc = subprocess_run( - f"{self._authenticator} true 3<&0", - input=f"{self._username}\0{self._password}\0\0".encode(), - shell=True, - timeout=2, - check=False, - ) - if completed_proc.returncode == 0: - return True - elif completed_proc.returncode == 1: - return False - else: - raise QMAuthError("authentication error", completed_proc.returncode) - except TimeoutExpired: - return False + self._pid = None + self._socket = None + self._client = None + self._connection = None def read_headers_for(self, folder, start, end, sort): - req = pb2.ListReq(folder=folder, start=start, end=end, sort=sort) - resp = self.build_and_run("list", req.SerializeToString()) - r = pb2.ListResp() - r.ParseFromString(resp) + sort_val = dict() + if sort[0] == "!": + sort = sort[1:] + sort_val["direction"] = "desc" + else: + sort_val["direction"] = "asc" + + match sort: + case "date" | "subject" | "size": + sort_val["parameter"] = sort + case _: + raise ValueError(f"invalid sort parameter {sort!r}") + + req = self._connection.List(folder=folder, start=start, end=end, sort=sort_val) return [ { - "message_handle": lmh.mid, - "byte_size": lmh.byte_size, - "unread": lmh.unread, - "date_received": lmh.rec_date, - "head": self._mail_header(lmh.header), + "message_handle": lmh["mid"], + "byte_size": lmh["byte_size"], + "unread": lmh["unread"], + "date_received": lmh["rec_date"], + "head": self._mail_header(lmh["header"]), } - for lmh in r.mail_heads + for lmh in req["mail_heads"] ] def count(self, folder): - req = pb2.StatsReq(folder=folder) - resp = self.build_and_run("count", req.SerializeToString()) - r = pb2.StatsResp() - r.ParseFromString(resp) + resp = self._connection.Stats(folder=folder) return { - "byte_size": r.byte_size, - "total_mails": r.mail_count, - "unread_mails": r.unread_count, + "byte_size": resp["byte_size"], + "total_mails": resp["mail_count"], + "unread_mails": resp["unread_count"], } def show(self, folder, msgid): - req = pb2.ShowReq(folder=folder, mid=msgid) - resp = self.build_and_run("read", req.SerializeToString()) - r = pb2.ShowResp() - r.ParseFromString(resp) + resp = self._connection.Show(folder=folder, mid=msgid) return { - "head": self._mail_header(r.mail.head), - "body": self._mail_body(r.mail.body), + "head": self._mail_header(resp["mail"]["head"]), + "body": self._mail_body(resp["mail"]["body"]), } def raw(self, folder, mid, path): - req = pb2.RawReq(folder=folder, mid=mid, path=path) - resp = self.build_and_run("raw", req.SerializeToString()) - r = pb2.RawResp() - r.ParseFromString(resp) - return {"head": self._mime_header(r.header), "body": r.body} + resp = self._connection.Raw(folder=folder, mid=mid, path=path) + return { + "head": self._mime_header(resp["header"]), + "body": b64decode(resp["body"]), + } - # def search(self, pattern, folder): - req = pb2.SearchReq(folder=folder, pattern=pattern) - resp = self.build_and_run("search", req.SerializeToString()) - r = pb2.SearchResp() - r.ParseFromString(resp) - return r + resp = self._connection.Search(folder=folder, pattern=pattern) + return resp def folders(self): - resp = self.build_and_run("folders", pb2.FoldersReq().SerializeToString()) - result = pb2.FoldersResp() - result.ParseFromString(resp) - return list(result.folders) + [""] + resp = self._connection.Folders() + return list(resp["folders"]) + [""] def move(self, mid, from_f, to_f): - req = pb2.MoveReq(mid=mid, from_f=from_f, to_f=to_f) - resp = self.build_and_run("move", req.SerializeToString()) - r = pb2.MoveResp() - r.ParseFromString(resp) + self._connection.Move(mid=mid, from_folder=from_f, to_folder=to_f) return True def remove(self, folder, msgid): - req = pb2.RemoveReq(folder=folder, mid=msgid) - resp = self.build_and_run("remove", req.SerializeToString()) - r = pb2.RemoveResp() - r.ParseFromString(resp) + self._connection.Remove(folder=folder, mid=msgid) return True def add_folder(self, name): - req = pb2.AddFolderReq(name=name) - resp = self.build_and_run("add-folder", req.SerializeToString()) - r = pb2.AddFolderResp() - r.ParseFromString(resp) - return r.status + resp = self._connection.AddFolder(name=name) + return resp["status"] @staticmethod def _address(addr): if not addr: return None - a = {"address": addr.address} - if addr.HasField("name"): - a["display_name"] = addr.name + a = {"address": addr["address"]} + if addr.get("name"): + a["display_name"] = addr["name"] return a - @classmethod - def _mail_header(cls, mail_head): + def _mail_header(self, mail_head): h = mail_head return { - "date": h.send_date, - "sender": cls._address(h.sender), - "cc": [cls._address(x) for x in h.cc if x], - "bcc": [cls._address(x) for x in h.bcc if x], - "from": [cls._address(x) for x in h.written_from if x], - "reply_to": [cls._address(x) for x in h.reply_to if x], - "to": [cls._address(x) for x in h.send_to if x], - "subject": h.subject, - "comments": list(h.comments), - "keywords": list(h.keywords), - "mime": cls._mime_header(h.mime), + "date": h["send_date"], + "sender": self._address(h.get("sender")), + "cc": [self._address(x) for x in h["cc"] if x], + "bcc": [self._address(x) for x in h["bcc"] if x], + "from": [self._address(x) for x in h["written_from"] if x], + "reply_to": [self._address(x) for x in h["reply_to"] if x], + "to": [self._address(x) for x in h["send_to"] if x], + "subject": h["subject"], + "comments": list(h["comments"]), + "keywords": list(h["keywords"]), + "mime": self._mime_header(h["mime"]), } @staticmethod def _mime_header(mime_head): mh = { - "content_maintype": mime_head.maintype, - "content_subtype": mime_head.subtype, + "content_maintype": mime_head["mime_type"]["main_type"], + "content_subtype": mime_head["mime_type"]["sub_type"], } - if ( - mime_head.contentdispo - == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_NONE - ): - mh["content_disposition"] = None - elif ( - mime_head.contentdispo - == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_INLINE - ): - mh["content_disposition"] = "inline" - elif ( - mime_head.contentdispo - == pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT - ): - mh["content_disposition"] = "attachment" - - if mime_head.HasField("file_name"): - mh["filename"] = mime_head.file_name + match mime_head["content_dispo"]: + case "inline": + mh["content_disposition"] = "inline" + case "attachment": + mh["content_disposition"] = "attachment" + case "none": + mh["content_disposition"] = None + case invalid: + raise ValueError(f"invalid content disposition {invalid!r}") + + if "file_name" in mime_head: + mh["filename"] = mime_head["file_name"] return mh - @classmethod - def _mail_body(cls, body): - if body.WhichOneof("Body") == "discrete": - return body.discrete - elif body.WhichOneof("Body") == "multipart": + def _mail_body(self, body): + assert len(body) == 1 + + if "discrete" in body: + return body["discrete"] + elif "multipart" in body: res = {"parts": []} - for mp in body.multipart.parts: - r = {"head": cls._mime_header(mp.mime_header)} + for mp in body["multipart"]["parts"]: + r = {"head": self._mime_header(mp["mime_header"])} - if ( - mp.mime_header.contentdispo - != pb2.MIMEHeader.ContentDisposition.CONTENT_DISPOSITION_ATTACHMENT - ): - r["body"] = cls._mail_body(mp.body) + if mp["mime_header"]["content_dispo"] != "attachment": + r["body"] = self._mail_body(mp["body"]) res["parts"].append(r) - if body.multipart.HasField("preamble"): - res["preamble"] = body.multipart.preamble - if body.multipart.HasField("epilogue"): - res["epilogue"] = body.multipart.epilogue + if pr := body["multipart"].get("preamble"): + res["preamble"] = pr + if ep := body["multipart"].get("epilogue"): + res["epilogue"] = ep return res - elif body.WhichOneof("Body") == "mail": + elif "mail" in body: return { - "head": cls._mail_header(body.mail.head), - "body": cls._mail_body(body.mail.body), + "head": self._mail_header(body["mail"]["head"]), + "body": self._mail_body(body["mail"]["body"]), } else: assert False - def _build_arg(self, user_mail_addr, mode, rp): - idx = user_mail_addr.find("@") - user_name = user_mail_addr[:idx] - - cmdline = [ - "moveto3.py", - "-a", - self._authenticator, - str(rp), - self._prog, - self._mailbox_path, - self._virtual_user, - user_name, - mode, - ] - - return cmdline - - def _read_qmauth(self, cmd, args, rp, wp): - - with Popen(cmd, stdin=PIPE, stdout=PIPE, pass_fds=[rp], bufsize=0) as popen: - os.close(rp) - os.write(wp, f"{self._username}\0{self._password}\0\0".encode()) + def open(self): + (rp, wp) = os.pipe() + (sp, sc) = socketpair() + cmdline = [self._authenticator, self._prog] + if (pid := os.fork()) == 0: + # child os.close(wp) - r = popen.stdout.read(10) - if popen.poll(): - raise QMAuthError( - "qmail-authuser unexpectedly exited", popen.returncode, r - ) - assert r == b"OPEN\n" - popen.stdin.write(args) - popen.stdin.close() - inp = popen.stdout.readall() - - try: - popen.wait(timeout=2) - except TimeoutExpired: - popen.kill() - popen.poll() - - rc = popen.returncode + sp.close() + os.dup2(rp, 3) + os.dup2(sc.fileno(), 4) + os.environ["LISTEN_FDS"] = "2" + os.environ["LISTEN_FDNAMES"] = "auth:varlink" + os.environ["LISTEN_PID"] = str(os.getpid()) + os.execvp(self._authenticator, cmdline) + assert False + sc.close() + os.close(rp) + os.write(wp, f"{self._username}\0{self._password}\0\0".encode()) + os.close(wp) - if rc == 0: - return inp - elif rc == 3: - raise QMAuthError("error reported by extractor", 3, inp) - else: - raise QMAuthError("got unsuccessful return code by qmail-authuser", rc, inp) + self._pid = pid + self._socket = sp + self._client = varlink.Client() - def build_and_run(self, mode, args): - (rp, wp) = os.pipe() - cmd = self._build_arg(self._username, mode, rp) - return self._read_qmauth(cmd, args, rp, wp) + try: + self._connection = self._client.open( + "de.jmhoffmann.jwebmail.mail-storage", connection=sp + ) + except ConnectionResetError: + (pid, status) = os.waitpid(pid, os.WNOHANG) + if pid != 0: + raise QMAuthError(os.waitstatus_to_exitcode(status)) + else: + raise + + user = self._username[: self._username.index("@")] + self._connection.Init( + unix_user=self._virtual_user, + mailbox_path=os.path.join(self._mailbox_path, user), + ) + return self + + def close(self): + self._connection.close() + self._socket.close() + (pid, status) = os.waitpid(self._pid, 0) + assert pid == self._pid + rc = os.waitstatus_to_exitcode(status) + if rc != 0: + raise QMAuthError(rc) diff --git a/src/jwebmail/read_mails.py b/src/jwebmail/read_mails.py index 54bd139..7775b12 100644 --- a/src/jwebmail/read_mails.py +++ b/src/jwebmail/read_mails.py @@ -5,7 +5,7 @@ from os.path import join as path_join from flask import current_app, g from flask_login import UserMixin, current_user, login_user -from .model.read_mails import QMailAuthuser +from .model.read_mails import QMailAuthuser, QMAuthError EXPIRATION_SEC = 60 * 60 * 25 @@ -163,8 +163,15 @@ def _build_qma(username, password): def login(username, password): - if not _build_qma(username, password).verify_user(): - return False + try: + qma = _build_qma(username, password).open() + except QMAuthError as err: + if err.rc == 1: + return False + else: + raise + finally: + qma.close() r = _select_timeout_session() r.set(username, password) r.close() @@ -186,6 +193,6 @@ def get_read_mails_logged_in(): if "read_mails" in g: return g.read_mails - qma = _build_qma(current_user.get_id(), current_user.password) + qma = _build_qma(current_user.get_id(), current_user.password).open() g.read_mails = qma return qma diff --git a/src/jwebmail/render_mail.py b/src/jwebmail/render_mail.py index 7dc0824..f76c40a 100644 --- a/src/jwebmail/render_mail.py +++ b/src/jwebmail/render_mail.py @@ -66,7 +66,7 @@ def render_multipart(_subtype, content, path): for i, p in enumerate(parts): R += "
\n" if ( - not p["head"]["content_disposition"] + not p["head"].get("content_disposition") or p["head"]["content_disposition"].lower() == "none" or p["head"]["content_disposition"].lower() == "inline" ): @@ -107,7 +107,7 @@ def _format_header(category, value): for v in value: value = ( f'"{v["display_name"]}" <{v["address"]}>' - if v["display_name"] + if v.get("display_name") else v["address"] ) R += f"
{escape(value)}
\n" -- cgit v1.2.3