1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

#

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

#

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

#

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

#

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

#

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

#

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

#

    17

#   Update history:

    18

#

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

#

    31

#   RCS $Log: $

    32

#

    33

#

    34

# -----------------------------------------------------------------------------

    35

"""

    36

mki18n allows you to internationalize your software.  You can use it to

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files:

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

#

    81

    82

from __future__ import absolute_import

    83

from __future__ import print_function

    84

import os

    85

import sys

    86

import re

    87

from builtins import str as text

    88

import wx

    89

    90

    91

# -----------------------------------------------------------------------------

    92

# Global variables

    93

# ----------------

    94

#

    95

    96

__author__ = "Pierre Rouleau"

    97

__version__ = "$Revision: 1.5 $"

    98

    99

# -----------------------------------------------------------------------------

   100

   101

   102

def getSupportedLanguageDict(appname):

   103

    """

   104

    Returns dictionary with languages already supported

   105

    by given application

   106

   107

    param: appname:

   108

        name of application

   109

    """

   110

    languageDict = {}

   111

    ext = '.po'

   112

    files = [x for x in os.listdir('.')

   113

             if x.startswith(appname) and x.endswith(ext)]

   114

   115

    langs = [x.split(appname + '_')[1].split(ext)[0] for x in files]

   116

    for lang in langs:

   117

        languageDict[lang] = lang

   118

   119

    return languageDict

   120

   121

   122

def getlanguageDict():

   123

    languageDict = {}

   124

    getSupportedLanguageDict('Beremiz')

   125

    if wx.VERSION >= (3, 0, 0):

   126

        _app = wx.App()

   127

    else:

   128

        _app = wx.PySimpleApp()

   129

   130

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   131

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   132

        if i:

   133

            languageDict[i.CanonicalName] = i.Description

   134

   135

    return languageDict

   136

   137

   138

def verbosePrint(verbose, str):

   139

    if verbose:

   140

        print(str)

   141

   142

   143

def processCustomFiles(filein, fileout, regexp, prefix=''):

   144

    appfil_file = open(filein, 'r')

   145

    messages_file = open(fileout, 'r')

   146

    messages = messages_file.read()

   147

    messages_file.close()

   148

    messages_file = open(fileout, 'a')

   149

    messages_file.write('\n')

   150

    messages_file.write('#: %s\n' % prefix)

   151

    messages_file.write('\n')

   152

   153

    words_found = {}

   154

    for filepath in appfil_file.readlines():

   155

        code_file = open(filepath.strip(), 'r')

   156

        for match in regexp.finditer(code_file.read()):

   157

            word = match.group(1)

   158

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   159

                words_found[word] = True

   160

                messages_file.write('\n')

   161

                messages_file.write("msgid \"%s\"\n" % word)

   162

                messages_file.write("msgstr \"\"\n")

   163

        code_file.close()

   164

   165

    messages_file.close()

   166

    appfil_file.close()

   167

   168

   169

# -----------------------------------------------------------------------------

   170

# m a k e P O ( )         -- Build the Portable Object file for the application --

   171

# ^^^^^^^^^^^^^^^

   172

#

   173

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0):

   174

    """Build the Portable Object Template file for the application.

   175

   176

    makePO builds the .pot file for the application stored inside

   177

    a specified directory by running xgettext for all application source

   178

    files.  It finds the name of all files by looking for a file called 'app.fil'.

   179

    If this file does not exists, makePo raises an IOError exception.

   180

    By default the application domain (the application

   181

    name) is the same as the directory name but it can be overridden by the

   182

    'applicationDomain' argument.

   183

   184

    makePO always creates a new file called messages.pot.  If it finds files

   185

    of the form app_xx.po where 'app' is the application name and 'xx' is one

   186

    of the ISO 639 two-letter language codes, makePO resynchronizes those

   187

    files with the latest extracted strings (now contained in messages.pot).

   188

    This process updates all line location number in the language-specific

   189

    .po files and may also create new entries for translation (or comment out

   190

    some).  The .po file is not changed, instead a new file is created with

   191

    the .new extension appended to the name of the .po file.

   192

   193

    By default the function does not display what it is doing.  Set the

   194

    verbose argument to 1 to force it to print its commands.

   195

    """

   196

   197

    if applicationDomain is None:

   198

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   199

    else:

   200

        applicationName = applicationDomain

   201

    currentDir = os.getcwd()

   202

    os.chdir(applicationDirectoryPath)

   203

    filelist = 'app.fil'

   204

    if not os.path.exists(filelist):

   205

        raise IOError(2, 'No module file: %s' % filelist)

   206

   207

    fileout = 'messages.pot'

   208

    # Steps:

   209

    #  Use xgettext to parse all application modules

   210

    #  The following switches are used:

   211

    #

   212

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   213

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   214

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   215

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   216

    verbosePrint(verbose, cmd)

   217

    os.system(cmd)

   218

   219

    XSD_STRING_MODEL = re.compile(r"<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   220

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   221

   222

    XML_TC6_STRING_MODEL = re.compile(r"<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   223

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')

   224

   225

    # generate messages.po

   226

    cmd = 'msginit --no-wrap --no-translator -i %s -l en_US.UTF-8 -o messages.po' % (fileout)

   227

    verbosePrint(verbose, cmd)

   228

    os.system(cmd)

   229

   230

    languageDict = getSupportedLanguageDict(applicationName)

   231

   232

    for langCode in languageDict.keys():

   233

        if langCode == 'en':

   234

            pass

   235

        else:

   236

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   237

            if os.path.exists(langPOfileName):

   238

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   239

                verbosePrint(verbose, cmd)

   240

                os.system(cmd)

   241

    os.chdir(currentDir)

   242

   243

   244

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0):

   245

    """Concatenate one or several PO files with the application domain files.

   246

    """

   247

   248

    if applicationDomain is None:

   249

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   250

    else:

   251

        applicationName = applicationDomain

   252

    currentDir = os.getcwd()

   253

    os.chdir(applicationDirectoryPath)

   254

   255

    languageDict = getSupportedLanguageDict(applicationName)

   256

   257

    for langCode in languageDict.keys():

   258

        if langCode == 'en':

   259

            pass

   260

        else:

   261

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   262

            if os.path.exists(langPOfileName):

   263

                fileList = ''

   264

                for fileName in listOf_extraPo:

   265

                    fileList += ("%s_%s.po " % (fileName, langCode))

   266

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   267

                verbosePrint(verbose, cmd)

   268

                os.system(cmd)

   269

                if targetDir is None:

   270

                    pass

   271

                else:

   272

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   273

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir, applicationName, applicationName, langCode)

   274

                    verbosePrint(verbose, cmd)

   275

                    os.system(cmd)

   276

    os.chdir(currentDir)

   277

   278

   279

def makeMO(applicationDirectoryPath, targetDir='./locale', applicationDomain=None, verbose=0, forceEnglish=0):

   280

    """Compile the Portable Object files into the Machine Object stored in the right location.

   281

   282

    makeMO converts all translated language-specific PO files located inside

   283

    the  application directory into the binary .MO files stored inside the

   284

    LC_MESSAGES sub-directory for the found locale files.

   285

   286

    makeMO searches for all files that have a name of the form 'app_xx.po'

   287

    inside the application directory specified by the first argument.  The

   288

    'app' is the application domain name (that can be specified by the

   289

    applicationDomain argument or is taken from the directory name). The 'xx'

   290

    corresponds to one of the ISO 639 two-letter language codes.

   291

   292

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   293

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   294

    code.

   295

    """

   296

   297

    if targetDir is None:

   298

        targetDir = './locale'

   299

   300

    verbosePrint(verbose, "Target directory for .mo files is: %s" % targetDir)

   301

   302

    if applicationDomain is None:

   303

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   304

    else:

   305

        applicationName = applicationDomain

   306

    currentDir = os.getcwd()

   307

    os.chdir(applicationDirectoryPath)

   308

   309

    languageDict = getSupportedLanguageDict(applicationName)

   310

   311

    for langCode in languageDict.keys():

   312

        if (langCode == 'en') and (forceEnglish == 0):

   313

            pass

   314

        else:

   315

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   316

            if os.path.exists(langPOfileName):

   317

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   318

                if not os.path.exists(mo_targetDir):

   319

                    mkdir(mo_targetDir)

   320

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir, applicationName, applicationName, langCode)

   321

                verbosePrint(verbose, cmd)

   322

                os.system(cmd)

   323

    os.chdir(currentDir)

   324

   325

   326

def printUsage(errorMsg=None):

   327

    """Displays how to use this script from the command line."""

   328

    print("""

   329

    ##################################################################################

   330

    #   mki18n :   Make internationalization files.                                  #

   331

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   332

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   333

    #              Supports C,C++,Python source files.                               #

   334

    #                                                                                #

   335

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   336

    #                                                                                #

   337

    #   Options:                                                                     #

   338

    #     -e               : When -m is used, forces English .mo file creation       #

   339

    #     -h               : prints this help                                        #

   340

    #     -m               : make MO from existing PO files                          #

   341

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   342

    #                        file. Creates a dom_xx.po.new for every existing        #

   343

    #                        language specific .po file. ('xx' stands for the ISO639 #

   344

    #                        two-letter language code and 'dom' stands for the       #

   345

    #                        application domain name).  mki18n requires that you     #

   346

    #                        write a 'app.fil' file  which contains the list of all  #

   347

    #                        source code to parse.                                   #

   348

    #     -v               : verbose (prints comments while running)                 #

   349

    #     --domain=appName : specifies the application domain name.  By default      #

   350

    #                        the directory name is used.                             #

   351

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   352

    #                      If not specified, the target is './locale'                #

   353

    #                                                                                #

   354

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   355

    #   specify the path of the target application.  If you leave it out mki18n      #

   356

    #   will use the current directory as the application main directory.            #

   357

    #                                                                                #

   358

    ##################################################################################""")

   359

    if errorMsg:

   360

        print("\n   ERROR: %s" % errorMsg)

   361

   362

   363

def fileBaseOf(filename, withPath=0):

   364

    """fileBaseOf(filename,withPath) ---> string

   365

   366

    Return base name of filename.  The returned string never includes the extension.

   367

    Use os.path.basename() to return the basename with the extension.  The

   368

    second argument is optional.  If specified and if set to 'true' (non zero)

   369

    the string returned contains the full path of the file name.  Otherwise the

   370

    path is excluded.

   371

   372

    [Example]

   373

    >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   374

    >>> fileBaseOf(fn)

   375

    'test'

   376

    >>> fileBaseOf(fn)

   377

    'test'

   378

    >>> fileBaseOf(fn,1)

   379

    'd:/dev/telepath/tvapp/code/test'

   380

    >>> fileBaseOf(fn,0)

   381

    'test'

   382

    >>> fn = 'abcdef'

   383

    >>> fileBaseOf(fn)

   384

    'abcdef'

   385

    >>> fileBaseOf(fn,1)

   386

    'abcdef'

   387

    >>> fn = "abcdef."

   388

    >>> fileBaseOf(fn)

   389

    'abcdef'

   390

    >>> fileBaseOf(fn,1)

   391

    'abcdef'

   392

    """

   393

    pos = filename.rfind('.')

   394

    if pos > 0:

   395

        filename = filename[:pos]

   396

    if withPath:

   397

        return filename

   398

    else:

   399

        return os.path.basename(filename)

   400

   401

   402

def mkdir(directory):

   403

    """Create a directory (and possibly the entire tree).

   404

   405

    The os.mkdir() will fail to create a directory if one of the

   406

    directory in the specified path does not exist.  mkdir()

   407

    solves this problem.  It creates every intermediate directory

   408

    required to create the final path. Under Unix, the function

   409

    only supports forward slash separator, but under Windows and MacOS

   410

    the function supports the forward slash and the OS separator (backslash

   411

    under windows).

   412

    """

   413

   414

    # translate the path separators

   415

    directory = unixpath(directory)

   416

    # build a list of all directory elements

   417

    aList = filter(lambda x: len(x) > 0, directory.split('/'))

   418

    theLen = len(aList)

   419

    # if the first element is a Windows-style disk drive

   420

    # concatenate it with the first directory

   421

    if aList[0].endswith(':'):

   422

        if theLen > 1:

   423

            aList[1] = aList[0] + '/' + aList[1]

   424

            del aList[0]

   425

            theLen -= 1

   426

    # if the original directory starts at root,

   427

    # make sure the first element of the list

   428

    # starts at root too

   429

    if directory[0] == '/':

   430

        aList[0] = '/' + aList[0]

   431

    # Now iterate through the list, check if the

   432

    # directory exists and if not create it

   433

    theDir = ''

   434

    for i in range(theLen):

   435

        theDir += aList[i]

   436

        if not os.path.exists(theDir):

   437

            os.mkdir(theDir)

   438

        theDir += '/'

   439

   440

   441

def unixpath(thePath):

   442

    r"""Return a path name that contains Unix separator.

   443

   444

    [Example]

   445

    >>> unixpath(r"d:\test")

   446

    'd:/test'

   447

    >>> unixpath("d:/test/file.txt")

   448

    'd:/test/file.txt'

   449

    >>>

   450

    """

   451

    thePath = os.path.normpath(thePath)

   452

    if os.sep == '/':

   453

        return thePath

   454

    else:

   455

        return thePath.replace(os.sep, '/')

   456

   457

   458

# -----------------------------------------------------------------------------

   459

   460

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   461

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   462

#

   463

if __name__ == "__main__":

   464

    # command line parsing

   465

    import getopt    # pylint: disable=wrong-import-order,wrong-import-position

   466

    argc = len(sys.argv)

   467

    if argc == 1:

   468

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   469

        sys.exit(1)

   470

    # If there is some arguments, parse the command line

   471

    validOptions = "ehmpv"

   472

    validLongOptions = ['domain=', 'moTarget=']

   473

    option = {}

   474

    option['forceEnglish'] = 0

   475

    option['mo'] = 0

   476

    option['po'] = 0

   477

    option['verbose'] = 0

   478

    option['domain'] = None

   479

    option['moTarget'] = None

   480

    optionKey = {

   481

        '-e':         'forceEnglish',

   482

        '-m':         'mo',

   483

        '-p':         'po',

   484

        '-v':         'verbose',

   485

        '--domain':   'domain',

   486

        '--moTarget': 'moTarget',

   487

    }

   488

    exit_code = 1

   489

    try: