1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

#

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

#

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

#

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

#

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

#

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

#

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

#

    17

#   Update history:

    18

#

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

#

    31

#   RCS $Log: $

    32

#

    33

#

    34

# -----------------------------------------------------------------------------

    35

"""

    36

mki18n allows you to internationalize your software.  You can use it to

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files:

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

#

    81

    82

from __future__ import absolute_import

    83

from __future__ import print_function

    84

import os

    85

import sys

    86

import re

    87

import wx

    88

    89

# -----------------------------------------------------------------------------

    90

# Global variables

    91

# ----------------

    92

#

    93

    94

__author__ = "Pierre Rouleau"

    95

__version__ = "$Revision: 1.5 $"

    96

    97

# -----------------------------------------------------------------------------

    98

    99

   100

def getSupportedLanguageDict(appname):

   101

    """

   102

    Returns dictionary with languages already supported

   103

    by given application

   104

   105

    param: appname:

   106

        name of application

   107

    """

   108

    languageDict = {}

   109

    ext = '.po'

   110

    files = [x for x in os.listdir('.')

   111

             if x.startswith(appname) and x.endswith(ext)]

   112

   113

    langs = [x.split(appname + '_')[1].split(ext)[0] for x in files]

   114

    for lang in langs:

   115

        languageDict[lang] = lang

   116

   117

    return languageDict

   118

   119

   120

def getlanguageDict():

   121

    languageDict = {}

   122

    getSupportedLanguageDict('Beremiz')

   123

    if wx.VERSION >= (3, 0, 0):

   124

        _app = wx.App()

   125

    else:

   126

        _app = wx.PySimpleApp()

   127

   128

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   129

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   130

        if i:

   131

            languageDict[i.CanonicalName] = i.Description

   132

   133

    return languageDict

   134

   135

   136

def verbosePrint(verbose, str):

   137

    if verbose:

   138

        print(str)

   139

   140

   141

def processCustomFiles(filein, fileout, regexp, prefix=''):

   142

    appfil_file = open(filein, 'r')

   143

    messages_file = open(fileout, 'r')

   144

    messages = messages_file.read()

   145

    messages_file.close()

   146

    messages_file = open(fileout, 'a')

   147

    messages_file.write('\n')

   148

    messages_file.write('#: %s\n' % prefix)

   149

    messages_file.write('\n')

   150

   151

    words_found = {}

   152

    for filepath in appfil_file.xreadlines():

   153

        code_file = open(filepath.strip(), 'r')

   154

        for match in regexp.finditer(code_file.read()):

   155

            word = match.group(1)

   156

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   157

                words_found[word] = True

   158

                messages_file.write('\n')

   159

                messages_file.write("msgid \"%s\"\n" % word)

   160

                messages_file.write("msgstr \"\"\n")

   161

        code_file.close()

   162

   163

    messages_file.close()

   164

    appfil_file.close()

   165

   166

   167

# -----------------------------------------------------------------------------

   168

# m a k e P O ( )         -- Build the Portable Object file for the application --

   169

# ^^^^^^^^^^^^^^^

   170

#

   171

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0):

   172

    """Build the Portable Object Template file for the application.

   173

   174

    makePO builds the .pot file for the application stored inside

   175

    a specified directory by running xgettext for all application source

   176

    files.  It finds the name of all files by looking for a file called 'app.fil'.

   177

    If this file does not exists, makePo raises an IOError exception.

   178

    By default the application domain (the application

   179

    name) is the same as the directory name but it can be overridden by the

   180

    'applicationDomain' argument.

   181

   182

    makePO always creates a new file called messages.pot.  If it finds files

   183

    of the form app_xx.po where 'app' is the application name and 'xx' is one

   184

    of the ISO 639 two-letter language codes, makePO resynchronizes those

   185

    files with the latest extracted strings (now contained in messages.pot).

   186

    This process updates all line location number in the language-specific

   187

    .po files and may also create new entries for translation (or comment out

   188

    some).  The .po file is not changed, instead a new file is created with

   189

    the .new extension appended to the name of the .po file.

   190

   191

    By default the function does not display what it is doing.  Set the

   192

    verbose argument to 1 to force it to print its commands.

   193

    """

   194

   195

    if applicationDomain is None:

   196

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   197

    else:

   198

        applicationName = applicationDomain

   199

    currentDir = os.getcwd()

   200

    os.chdir(applicationDirectoryPath)

   201

    filelist = 'app.fil'

   202

    if not os.path.exists(filelist):

   203

        raise IOError(2, 'No module file: ' % filelist)

   204

   205

    fileout = 'messages.pot'

   206

    # Steps:

   207

    #  Use xgettext to parse all application modules

   208

    #  The following switches are used:

   209

    #

   210

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   211

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   212

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   213

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   214

    verbosePrint(verbose, cmd)

   215

    os.system(cmd)

   216

   217

    XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   218

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   219

   220

    XML_TC6_STRING_MODEL = re.compile("<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   221

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')

   222

   223

    # generate messages.po

   224

    cmd = 'msginit --no-wrap --no-translator -i %s -l en_US.UTF-8 -o messages.po' % (fileout)

   225

    verbosePrint(verbose, cmd)

   226

    os.system(cmd)

   227

   228

    languageDict = getSupportedLanguageDict(applicationName)

   229

   230

    for langCode in languageDict.keys():

   231

        if langCode == 'en':

   232

            pass

   233

        else:

   234

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   235

            if os.path.exists(langPOfileName):

   236

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   237

                verbosePrint(verbose, cmd)

   238

                os.system(cmd)

   239

    os.chdir(currentDir)

   240

   241

   242

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0):

   243

    """Concatenate one or several PO files with the application domain files.

   244

    """

   245

   246

    if applicationDomain is None:

   247

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   248

    else:

   249

        applicationName = applicationDomain

   250

    currentDir = os.getcwd()

   251

    os.chdir(applicationDirectoryPath)

   252

   253

    languageDict = getSupportedLanguageDict(applicationName)

   254

   255

    for langCode in languageDict.keys():

   256

        if langCode == 'en':

   257

            pass

   258

        else:

   259

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   260

            if os.path.exists(langPOfileName):

   261

                fileList = ''

   262

                for fileName in listOf_extraPo:

   263

                    fileList += ("%s_%s.po " % (fileName, langCode))

   264

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   265

                verbosePrint(verbose, cmd)

   266

                os.system(cmd)

   267

                if targetDir is None:

   268

                    pass

   269

                else:

   270

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   271

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir, applicationName, applicationName, langCode)

   272

                    verbosePrint(verbose, cmd)

   273

                    os.system(cmd)

   274

    os.chdir(currentDir)

   275

   276

   277

def makeMO(applicationDirectoryPath, targetDir='./locale', applicationDomain=None, verbose=0, forceEnglish=0):

   278

    """Compile the Portable Object files into the Machine Object stored in the right location.

   279

   280

    makeMO converts all translated language-specific PO files located inside

   281

    the  application directory into the binary .MO files stored inside the

   282

    LC_MESSAGES sub-directory for the found locale files.

   283

   284

    makeMO searches for all files that have a name of the form 'app_xx.po'

   285

    inside the application directory specified by the first argument.  The

   286

    'app' is the application domain name (that can be specified by the

   287

    applicationDomain argument or is taken from the directory name). The 'xx'

   288

    corresponds to one of the ISO 639 two-letter language codes.

   289

   290

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   291

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   292

    code.

   293

    """

   294

   295

    if targetDir is None:

   296

        targetDir = './locale'

   297

   298

    verbosePrint(verbose, "Target directory for .mo files is: %s" % targetDir)

   299

   300

    if applicationDomain is None:

   301

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   302

    else:

   303

        applicationName = applicationDomain

   304

    currentDir = os.getcwd()

   305

    os.chdir(applicationDirectoryPath)

   306

   307

    languageDict = getSupportedLanguageDict(applicationName)

   308

   309

    for langCode in languageDict.keys():

   310

        if (langCode == 'en') and (forceEnglish == 0):

   311

            pass

   312

        else:

   313

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   314

            if os.path.exists(langPOfileName):

   315

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   316

                if not os.path.exists(mo_targetDir):

   317

                    mkdir(mo_targetDir)

   318

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir, applicationName, applicationName, langCode)

   319

                verbosePrint(verbose, cmd)

   320

                os.system(cmd)

   321

    os.chdir(currentDir)

   322

   323

   324

def printUsage(errorMsg=None):

   325

    """Displays how to use this script from the command line."""

   326

    print("""

   327

    ##################################################################################

   328

    #   mki18n :   Make internationalization files.                                  #

   329

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   330

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   331

    #              Supports C,C++,Python source files.                               #

   332

    #                                                                                #

   333

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   334

    #                                                                                #

   335

    #   Options:                                                                     #

   336

    #     -e               : When -m is used, forces English .mo file creation       #

   337

    #     -h               : prints this help                                        #

   338

    #     -m               : make MO from existing PO files                          #

   339

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   340

    #                        file. Creates a dom_xx.po.new for every existing        #

   341

    #                        language specific .po file. ('xx' stands for the ISO639 #

   342

    #                        two-letter language code and 'dom' stands for the       #

   343

    #                        application domain name).  mki18n requires that you     #

   344

    #                        write a 'app.fil' file  which contains the list of all  #

   345

    #                        source code to parse.                                   #

   346

    #     -v               : verbose (prints comments while running)                 #

   347

    #     --domain=appName : specifies the application domain name.  By default      #

   348

    #                        the directory name is used.                             #

   349

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   350

    #                      If not specified, the target is './locale'                #

   351

    #                                                                                #

   352

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   353

    #   specify the path of the target application.  If you leave it out mki18n      #

   354

    #   will use the current directory as the application main directory.            #

   355

    #                                                                                #

   356

    ##################################################################################""")

   357

    if errorMsg:

   358

        print("\n   ERROR: %s" % errorMsg)

   359

   360

   361

def fileBaseOf(filename, withPath=0):

   362

    """fileBaseOf(filename,withPath) ---> string

   363

   364

    Return base name of filename.  The returned string never includes the extension.

   365

    Use os.path.basename() to return the basename with the extension.  The

   366

    second argument is optional.  If specified and if set to 'true' (non zero)

   367

    the string returned contains the full path of the file name.  Otherwise the

   368

    path is excluded.

   369

   370

    [Example]

   371

    >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   372

    >>> fileBaseOf(fn)

   373

    'test'

   374

    >>> fileBaseOf(fn)

   375

    'test'

   376

    >>> fileBaseOf(fn,1)

   377

    'd:/dev/telepath/tvapp/code/test'

   378

    >>> fileBaseOf(fn,0)

   379

    'test'

   380

    >>> fn = 'abcdef'

   381

    >>> fileBaseOf(fn)

   382

    'abcdef'

   383

    >>> fileBaseOf(fn,1)

   384

    'abcdef'

   385

    >>> fn = "abcdef."

   386

    >>> fileBaseOf(fn)

   387

    'abcdef'

   388

    >>> fileBaseOf(fn,1)

   389

    'abcdef'

   390

    """

   391

    pos = filename.rfind('.')

   392

    if pos > 0:

   393

        filename = filename[:pos]

   394

    if withPath:

   395

        return filename

   396

    else:

   397

        return os.path.basename(filename)

   398

   399

   400

def mkdir(directory):

   401

    """Create a directory (and possibly the entire tree).

   402

   403

    The os.mkdir() will fail to create a directory if one of the

   404

    directory in the specified path does not exist.  mkdir()

   405

    solves this problem.  It creates every intermediate directory

   406

    required to create the final path. Under Unix, the function

   407

    only supports forward slash separator, but under Windows and MacOS

   408

    the function supports the forward slash and the OS separator (backslash

   409

    under windows).

   410

    """

   411

   412

    # translate the path separators

   413

    directory = unixpath(directory)

   414

    # build a list of all directory elements

   415

    aList = filter(lambda x: len(x) > 0, directory.split('/'))

   416

    theLen = len(aList)

   417

    # if the first element is a Windows-style disk drive

   418

    # concatenate it with the first directory

   419

    if aList[0].endswith(':'):

   420

        if theLen > 1:

   421

            aList[1] = aList[0] + '/' + aList[1]

   422

            del aList[0]

   423

            theLen -= 1

   424

    # if the original directory starts at root,

   425

    # make sure the first element of the list

   426

    # starts at root too

   427

    if directory[0] == '/':

   428

        aList[0] = '/' + aList[0]

   429

    # Now iterate through the list, check if the

   430

    # directory exists and if not create it

   431

    theDir = ''

   432

    for i in range(theLen):

   433

        theDir += aList[i]

   434

        if not os.path.exists(theDir):

   435

            os.mkdir(theDir)

   436

        theDir += '/'

   437

   438

   439

def unixpath(thePath):

   440

    r"""Return a path name that contains Unix separator.

   441

   442

    [Example]

   443

    >>> unixpath(r"d:\test")

   444

    'd:/test'

   445

    >>> unixpath("d:/test/file.txt")

   446

    'd:/test/file.txt'

   447

    >>>

   448

    """

   449

    thePath = os.path.normpath(thePath)

   450

    if os.sep == '/':

   451

        return thePath

   452

    else:

   453

        return thePath.replace(os.sep, '/')

   454

   455

   456

# -----------------------------------------------------------------------------

   457

   458

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   459

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   460

#

   461

if __name__ == "__main__":

   462

    # command line parsing

   463

    import getopt    # pylint: disable=wrong-import-order,wrong-import-position

   464

    argc = len(sys.argv)

   465

    if argc == 1:

   466

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   467

        sys.exit(1)

   468

    # If there is some arguments, parse the command line

   469

    validOptions = "ehmpv"

   470

    validLongOptions = ['domain=', 'moTarget=']

   471

    option = {}

   472

    option['forceEnglish'] = 0

   473

    option['mo'] = 0

   474

    option['po'] = 0

   475

    option['verbose'] = 0

   476

    option['domain'] = None

   477

    option['moTarget'] = None

   478

    optionKey = {

   479

        '-e':         'forceEnglish',

   480

        '-m':         'mo',

   481

        '-p':         'po',

   482

        '-v':         'verbose',

   483

        '--domain':   'domain',

   484

        '--moTarget': 'moTarget',

   485

    }

   486

    exit_code = 1

   487

    try:

   488

        optionList, pargs = getopt.getopt(sys.argv[1:], validOptions, validLongOptions)

   489

    except getopt.GetoptError as e:

   490