Edouard@3112: #!/usr/bin/env python
Edouard@3112: # -*- coding: utf-8 -*-
Edouard@3112: 
Edouard@3112: # This file is part of Beremiz
Edouard@3112: # Copyright (C) 2021: Edouard TISSERANT
Edouard@3112: #
Edouard@3112: # See COPYING file for copyrights details.
Edouard@3112: 
kinsamanka@3750: 
Edouard@3115: from lxml import etree
edouard@3113: import os
Edouard@3112: import sys
Edouard@3112: import subprocess
edouard@3108: import time
edouard@3113: import ast
Edouard@3112: import wx
Edouard@3157: import re
edouard@3915: from email.parser import HeaderParser
Edouard@3112: 
edouard@3133: import pycountry
edouard@3573: from dialogs import MessageBoxOnce
edouard@3915: from POULibrary import UserAddressedException
edouard@3133: 
Edouard@3157: cmd_parser = re.compile(r'(?:"([^"]+)"\s*|([^\s]+)\s*)?')
Edouard@3157: 
Edouard@3112: def open_pofile(pofile):
Edouard@3112:     """ Opens PO file with POEdit """
Edouard@3112:     
Edouard@3112:     if sys.platform.startswith('win'):
edouard@3919:         import winreg
Edouard@3112:         poedit_cmd = None
Edouard@3112:         try:
Edouard@3112:             poedit_cmd = winreg.QueryValue(winreg.HKEY_LOCAL_MACHINE,
Edouard@3112:                                            'SOFTWARE\\Classes\\poedit\\shell\\open\\command')
Edouard@3157:             cmd = re.findall(cmd_parser, poedit_cmd)
Edouard@3157:             dblquote_value,smpl_value = cmd[0]
Edouard@3157:             poedit_path = dblquote_value+smpl_value
Edouard@3112:         except OSError:
Edouard@3112:             poedit_path = None
Edouard@3112: 
Edouard@3112:     else:
kinsamanka@3750:         if "SNAP" in os.environ:
edouard@3573:             MessageBoxOnce("Launching POEdit with xdg-open",
edouard@3573:                     "Confined app can't launch POEdit directly.\n"+
edouard@3573:                         "Instead, PO/POT file is passed to xdg-open.\n"+
edouard@3573:                         "Please select POEdit when proposed.\n\n"+
edouard@3573:                     "Notes: \n"+
edouard@3573:                     " - POEdit must be installed on you system.\n"+
edouard@3573:                     " - If no choice is proposed, use file manager to change POT/PO file properties.\n",
edouard@3573:                     "SVGHMII18SnapWarning")
edouard@3573:             poedit_path = "xdg-open"
edouard@3573:         else:
edouard@3573:             try:
edouard@3573:                 poedit_path = subprocess.check_output("command -v poedit", shell=True).strip()
edouard@3573:             except subprocess.CalledProcessError:
edouard@3573:                 poedit_path = None
Edouard@3112: 
Edouard@3112:     if poedit_path is None:
Edouard@3112:         wx.MessageBox("POEdit is not found or installed !")
Edouard@3112:     else:
Edouard@3112:         subprocess.Popen([poedit_path,pofile])
edouard@3108: 
edouard@3113: def EtreeToMessages(msgs):
edouard@3113:     """ Converts XML tree from 'extract_i18n' templates into a list of tuples """
edouard@3113:     messages = []
edouard@3113: 
edouard@3113:     for msg in msgs:
edouard@3113:         messages.append((
edouard@3915:             b"\n".join([line.text.encode() for line in msg]),
edouard@3915:             msg.get("label").encode(), msg.get("id").encode()))
edouard@3113: 
edouard@3113:     return messages
edouard@3113: 
edouard@3113: def SaveCatalog(fname, messages):
edouard@3113:     """ Save messages given as list of tupple (msg,label,id) in POT file """
edouard@3113:     w = POTWriter()
edouard@3113:     w.ImportMessages(messages)
edouard@3113: 
edouard@3915:     with open(fname, 'wb') as POT_file:
edouard@3113:         w.write(POT_file)
Edouard@3114: 
edouard@3214: def GetPoFiles(dirpath):
edouard@3214:     po_files = [fname for fname in os.listdir(dirpath) if fname.endswith(".po")]
edouard@3214:     po_files.sort()
Edouard@3218:     return [(po_fname[:-3],os.path.join(dirpath, po_fname)) for po_fname in po_files]
edouard@3214: 
edouard@3113: def ReadTranslations(dirpath):
edouard@3144:     """ Read all PO files from a directory and return a list of (langcode, translation_dict) tuples """
edouard@3113: 
edouard@3113:     translations = []
Edouard@3218:     for translation_name, po_path in GetPoFiles(dirpath):
edouard@3919:         messages = POReader().read(po_path)
edouard@3919:         translations.append((translation_name, messages))
edouard@3113:     return translations
edouard@3113: 
edouard@3113: def MatchTranslations(translations, messages, errcallback):
Edouard@3114:     """
Edouard@3114:     Matches translations against original message catalog,
Edouard@3114:     warn about inconsistancies,
Edouard@3114:     returns list of langs, and a list of (msgid, [translations]) tuples
edouard@3113:     """
edouard@3113:     translated_messages = []
Edouard@3114:     broken_lang = set()
edouard@3919:     incomplete_lang = set()
edouard@3113:     for msgid,label,svgid in messages:
edouard@3113:         translated_message = []
edouard@3144:         for langcode,translation in translations:
edouard@3113:             msg = translation.pop(msgid, None)
edouard@3113:             if msg is None:
edouard@3919:                 # Missing translation (msgid not in .po)
edouard@3144:                 broken_lang.add(langcode)
edouard@3919:                 msg = msgid
edouard@3919:             elif msg == b"":
edouard@3919:                 # Empty translation (msgid is in .po)
edouard@3919:                 incomplete_lang.add(langcode)
edouard@3919:                 msg = msgid
edouard@3919:             
edouard@3919:             translated_message.append(msg)
edouard@3113:         translated_messages.append((msgid,translated_message))
edouard@3113:     langs = []
edouard@3144:     for langcode,translation in translations:
edouard@3133:         try:
edouard@3144:             l,c = langcode.split("_")
edouard@3133:             language_name = pycountry.languages.get(alpha_2 = l).name
edouard@3133:             country_name = pycountry.countries.get(alpha_2 = c).name
edouard@3144:             langname = "{} ({})".format(language_name, country_name)
edouard@3133:         except:
edouard@3133:             try:
edouard@3144:                 langname = pycountry.languages.get(alpha_2 = langcode).name
edouard@3133:             except:
edouard@3144:                 langname = langcode
edouard@3144: 
edouard@3144:         langs.append((langname,langcode))
edouard@3133: 
Edouard@3114:         broken = False
edouard@3919:         if translation:
Edouard@3115:             broken = True
edouard@3144:         if broken or langcode in broken_lang:
edouard@3919:             errcallback((
edouard@3919:                     _('Translation for {} is outdated and incomplete.') 
edouard@3919:                     if langcode in incomplete_lang else
edouard@3919:                     _('Translation for {} is outdated.')).format(langcode) + 
edouard@3919:                 " "+_('Edit {}.po, click "Catalog -> Update from POT File..." and select messages.pot.\n').format(langcode))
edouard@3919:         elif langcode in incomplete_lang:
edouard@3919:             errcallback(_('Translation for {} is incomplete. Edit {}.po to complete it.\n').format(langcode,langcode))
Edouard@3114: 
edouard@3113: 
edouard@3113:     return langs,translated_messages
edouard@3113: 
Edouard@3114: 
edouard@3113: def TranslationToEtree(langs,translated_messages):
Edouard@3115: 
Edouard@3116:     result = etree.Element("translations")
Edouard@3116: 
Edouard@3116:     langsroot = etree.SubElement(result, "langs")
edouard@3144:     for name, code in langs:
edouard@3144:         langel = etree.SubElement(langsroot, "lang", {"code":code})
edouard@3144:         langel.text = name
Edouard@3115: 
Edouard@3116:     msgsroot = etree.SubElement(result, "messages")
Edouard@3115:     for msgid, msgs in translated_messages:
Edouard@3115:         msgidel = etree.SubElement(msgsroot, "msgid")
Edouard@3115:         for msg in msgs:
Edouard@3115:             msgel = etree.SubElement(msgidel, "msg")
edouard@3915:             for line in msg.split(b"\n"):
Edouard@3116:                 lineel = etree.SubElement(msgel, "line")
edouard@3915:                 lineel.text = escape(line).decode()
Edouard@3116: 
Edouard@3116:     return result
Edouard@3115: 
edouard@3915: # Code below is based on :
edouard@3915: #  cpython/Tools/i18n/pygettext.py
edouard@3915: #  cpython/Tools/i18n/msgfmt.py
edouard@3915: 
edouard@3915: locpfx = b'#:svghmi.svg:'
edouard@3108: 
edouard@3108: pot_header = '''\
edouard@3108: # SOME DESCRIPTIVE TITLE.
edouard@3108: # Copyright (C) YEAR ORGANIZATION
edouard@3108: # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
edouard@3108: #
edouard@3108: msgid ""
edouard@3108: msgstr ""
edouard@3108: "Project-Id-Version: PACKAGE VERSION\\n"
edouard@3108: "POT-Creation-Date: %(time)s\\n"
edouard@3108: "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"
edouard@3108: "Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n"
edouard@3108: "Language-Team: LANGUAGE <LL@li.org>\\n"
edouard@3108: "MIME-Version: 1.0\\n"
edouard@3113: "Content-Type: text/plain; charset=UTF-8\\n"
edouard@3113: "Content-Transfer-Encoding: 8bit\\n"
edouard@3108: "Generated-By: SVGHMI 1.0\\n"
edouard@3108: 
edouard@3918: 
edouard@3108: '''
Edouard@3112: escapes = []
Edouard@3112: 
edouard@3915: def make_escapes():
Edouard@3112:     global escapes
edouard@3915:     escapes = [b"\%03o" % i for i in range(128)]
edouard@3915:     for i in range(32, 127):
edouard@3915:         escapes[i] = bytes([i])
edouard@3915:     escapes[ord('\\')] = b'\\\\'
edouard@3915:     escapes[ord('\t')] = b'\\t'
edouard@3915:     escapes[ord('\r')] = b'\\r'
edouard@3915:     escapes[ord('\n')] = b'\\n'
edouard@3915:     escapes[ord('\"')] = b'\\"'
edouard@3915: 
edouard@3915: make_escapes()
Edouard@3112: 
Edouard@3112: def escape(s):
edouard@3918:     return b''.join([escapes[c] if c < 128 else bytes([c]) for c in s])
Edouard@3112: 
Edouard@3112: def normalize(s):
Edouard@3112:     # This converts the various Python string types into a format that is
Edouard@3112:     # appropriate for .po files, namely much closer to C style.
edouard@3915:     lines = s.split(b'\n')
Edouard@3112:     if len(lines) == 1:
edouard@3915:         s = b'"' + escape(s) + b'"'
Edouard@3112:     else:
Edouard@3112:         if not lines[-1]:
Edouard@3112:             del lines[-1]
edouard@3915:             lines[-1] = lines[-1] + b'\n'
Edouard@3112:         for i in range(len(lines)):
Edouard@3112:             lines[i] = escape(lines[i])
edouard@3915:         lineterm = b'\\n"\n"'
edouard@3915:         s = b'""\n"' + lineterm.join(lines) + b'"'
Edouard@3112:     return s
Edouard@3112: 
edouard@3108: class POTWriter:
edouard@3108:     def __init__(self):
edouard@3108:         self.__messages = {}
edouard@3108: 
Edouard@3112:     def ImportMessages(self, msgs):
Edouard@3115:         for  msg, label, svgid in msgs:
edouard@3915:             self.addentry(msg, label, svgid)
edouard@3108: 
edouard@3108:     def addentry(self, msg, label, svgid):
edouard@3108:         entry = (label, svgid)
edouard@3108:         self.__messages.setdefault(msg, set()).add(entry)
edouard@3108: 
edouard@3108:     def write(self, fp):
edouard@3915:         timestamp = time.strftime('%Y-%m-%d %H:%M%z')
edouard@3915:         header = pot_header % {'time': timestamp}
edouard@3915:         fp.write(header.encode())
edouard@3108:         reverse = {}
edouard@3915:         for k, v in self.__messages.items():
edouard@3108:             keys = list(v)
edouard@3108:             keys.sort()
edouard@3108:             reverse.setdefault(tuple(keys), []).append((k, v))
edouard@3915:         rkeys = sorted(reverse.keys())
edouard@3108:         for rkey in rkeys:
edouard@3108:             rentries = reverse[rkey]
edouard@3108:             rentries.sort()
edouard@3108:             for k, v in rentries:
Edouard@3112:                 v = list(v)
edouard@3108:                 v.sort()
edouard@3108:                 locline = locpfx
edouard@3108:                 for label, svgid in v:
edouard@3915:                     d = {b'label': label, b'svgid': svgid}
edouard@3915:                     s = b' %(label)s:%(svgid)s' % d
edouard@3108:                     if len(locline) + len(s) <= 78:
edouard@3108:                         locline = locline + s
edouard@3108:                     else:
edouard@3918:                         fp.write(locline + b'\n')
edouard@3108:                         locline = locpfx + s
edouard@3108:                 if len(locline) > len(locpfx):
edouard@3918:                     fp.write(locline + b'\n')
edouard@3918:                 fp.write(b'msgid ' + normalize(k) + b'\n')
edouard@3918:                 fp.write(b'msgstr ""\n\n')
edouard@3108: 
edouard@3108: 
edouard@3108: class POReader:
edouard@3108:     def __init__(self):
edouard@3108:         self.__messages = {}
edouard@3108: 
edouard@3915:     def add(self, ctxt, msgid, msgstr, fuzzy):
edouard@3919:         "Add eventually empty translation to the dictionary."
edouard@3919:         if msgid:
edouard@3919:             self.__messages[msgid if ctxt is None else (b"%b\x04%b" % (ctxt, id))] = msgstr
edouard@3915: 
edouard@3915:     def read(self, infile):
edouard@3108:         ID = 1
edouard@3108:         STR = 2
edouard@3915:         CTXT = 3
edouard@3915: 
edouard@3919:         self.__messages = {}
edouard@3919: 
edouard@3919:         try:
edouard@3919:             with open(infile, 'rb') as f:
edouard@3919:                 lines = f.readlines()
edouard@3919:         except Exception as e:
edouard@3919:             raise UserAddressedException(
edouard@3919:                         'Cannot open PO translation file :%s' % str(e))
edouard@3915:             
edouard@3915:         section = msgctxt = None
edouard@3108:         fuzzy = 0
edouard@3108: 
edouard@3915:         # Start off assuming Latin-1, so everything decodes without failure,
edouard@3915:         # until we know the exact encoding
edouard@3915:         encoding = 'latin-1'
edouard@3915: 
edouard@3108:         # Parse the catalog
edouard@3108:         lno = 0
edouard@3108:         for l in lines:
edouard@3915:             l = l.decode(encoding)
edouard@3108:             lno += 1
edouard@3108:             # If we get a comment line after a msgstr, this is a new entry
edouard@3108:             if l[0] == '#' and section == STR:
edouard@3915:                 self.add(msgctxt, msgid, msgstr, fuzzy)
edouard@3915:                 section = msgctxt = None
edouard@3108:                 fuzzy = 0
edouard@3108:             # Record a fuzzy mark
edouard@3108:             if l[:2] == '#,' and 'fuzzy' in l:
edouard@3108:                 fuzzy = 1
edouard@3108:             # Skip comments
edouard@3108:             if l[0] == '#':
edouard@3108:                 continue
edouard@3915:             # Now we are in a msgid or msgctxt section, output previous section
edouard@3915:             if l.startswith('msgctxt'):
edouard@3108:                 if section == STR:
edouard@3915:                     self.add(msgctxt, msgid, msgstr, fuzzy)
edouard@3919:                     fuzzy = 0
edouard@3915:                 section = CTXT
edouard@3915:                 l = l[7:]
edouard@3915:                 msgctxt = b''
edouard@3915:             elif l.startswith('msgid') and not l.startswith('msgid_plural'):
edouard@3915:                 if section == STR:
edouard@3915:                     self.add(msgctxt, msgid, msgstr, fuzzy)
edouard@3919:                     fuzzy = 0
edouard@3915:                     if not msgid:
edouard@3915:                         # See whether there is an encoding declaration
edouard@3915:                         p = HeaderParser()
edouard@3915:                         charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
edouard@3915:                         if charset:
edouard@3915:                             encoding = charset
edouard@3108:                 section = ID
edouard@3108:                 l = l[5:]
edouard@3915:                 msgid = msgstr = b''
edouard@3108:                 is_plural = False
edouard@3108:             # This is a message with plural forms
edouard@3108:             elif l.startswith('msgid_plural'):
edouard@3108:                 if section != ID:
edouard@3915:                     raise UserAddressedException(
edouard@3915:                         'msgid_plural not preceded by msgid on %s:%d' % (infile, lno))
edouard@3108:                 l = l[12:]
edouard@3915:                 msgid += b'\0' # separator of singular and plural
edouard@3108:                 is_plural = True
edouard@3108:             # Now we are in a msgstr section
edouard@3108:             elif l.startswith('msgstr'):
edouard@3108:                 section = STR
edouard@3108:                 if l.startswith('msgstr['):
edouard@3108:                     if not is_plural:
edouard@3915:                         raise UserAddressedException(
edouard@3915:                             'plural without msgid_plural on %s:%d' % (infile, lno))
edouard@3108:                     l = l.split(']', 1)[1]
edouard@3108:                     if msgstr:
edouard@3915:                         msgstr += b'\0' # Separator of the various plural forms
edouard@3108:                 else:
edouard@3108:                     if is_plural:
edouard@3915:                         raise UserAddressedException(
edouard@3915:                             'indexed msgstr required for plural on  %s:%d' % (infile, lno))
edouard@3108:                     l = l[6:]
edouard@3108:             # Skip empty lines
edouard@3108:             l = l.strip()
edouard@3108:             if not l:
edouard@3108:                 continue
edouard@3108:             l = ast.literal_eval(l)
edouard@3915:             if section == CTXT:
edouard@3915:                 msgctxt += l.encode(encoding)
edouard@3915:             elif section == ID:
edouard@3915:                 msgid += l.encode(encoding)
edouard@3108:             elif section == STR:
edouard@3915:                 msgstr += l.encode(encoding)
edouard@3108:             else:
edouard@3915:                 raise UserAddressedException(
edouard@3915:                     'Syntax error on %s:%d' % (infile, lno) + 'before:\n %s'%l)
edouard@3108:         # Add last entry
edouard@3108:         if section == STR:
edouard@3915:             self.add(msgctxt, msgid, msgstr, fuzzy)
edouard@3915: 
edouard@3919:         return self.__messages
edouard@3919: 
edouard@3919: 
edouard@3919: