1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

#

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

#

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

#

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

#

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

#

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

#

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

#

    17

#   Update history:

    18

#

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

#

    31

#   RCS $Log: $

    32

#

    33

#

    34

# -----------------------------------------------------------------------------

    35

"""

    36

mki18n allows you to internationalize your software.  You can use it to

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files:

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

#

    81

    82

from __future__ import absolute_import

    83

from __future__ import print_function

    84

import os

    85

import sys

    86

import re

    87

import wx

    88

    89

# -----------------------------------------------------------------------------

    90

# Global variables

    91

# ----------------

    92

#

    93

    94

__author__ = "Pierre Rouleau"

    95

__version__ = "$Revision: 1.5 $"

    96

    97

# -----------------------------------------------------------------------------

    98

    99

   100

def getlanguageDict():

   101

    languageDict = {}

   102

   103

    if wx.VERSION >= (3, 0, 0):

   104

        _app = wx.App()

   105

    else:

   106

        _app = wx.PySimpleApp()

   107

   108

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   109

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   110

        if i:

   111

            languageDict[i.CanonicalName] = i.Description

   112

   113

    return languageDict

   114

   115

   116

def verbosePrint(verbose, str):

   117

    if verbose:

   118

        print(str)

   119

   120

   121

def processCustomFiles(filein, fileout, regexp, prefix=''):

   122

    appfil_file = open(filein, 'r')

   123

    messages_file = open(fileout, 'r')

   124

    messages = messages_file.read()

   125

    messages_file.close()

   126

    messages_file = open(fileout, 'a')

   127

    messages_file.write('\n')

   128

    messages_file.write('#: %s\n' % prefix)

   129

    messages_file.write('\n')

   130

   131

    words_found = {}

   132

    for filepath in appfil_file.xreadlines():

   133

        code_file = open(filepath.strip(), 'r')

   134

        for match in regexp.finditer(code_file.read()):

   135

            word = match.group(1)

   136

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   137

                words_found[word] = True

   138

                messages_file.write('\n')

   139

                messages_file.write("msgid \"%s\"\n" % word)

   140

                messages_file.write("msgstr \"\"\n")

   141

        code_file.close()

   142

   143

    messages_file.close()

   144

    appfil_file.close()

   145

   146

   147

# -----------------------------------------------------------------------------

   148

# m a k e P O ( )         -- Build the Portable Object file for the application --

   149

# ^^^^^^^^^^^^^^^

   150

#

   151

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0):

   152

    """Build the Portable Object Template file for the application.

   153

   154

    makePO builds the .pot file for the application stored inside

   155

    a specified directory by running xgettext for all application source

   156

    files.  It finds the name of all files by looking for a file called 'app.fil'.

   157

    If this file does not exists, makePo raises an IOError exception.

   158

    By default the application domain (the application

   159

    name) is the same as the directory name but it can be overridden by the

   160

    'applicationDomain' argument.

   161

   162

    makePO always creates a new file called messages.pot.  If it finds files

   163

    of the form app_xx.po where 'app' is the application name and 'xx' is one

   164

    of the ISO 639 two-letter language codes, makePO resynchronizes those

   165

    files with the latest extracted strings (now contained in messages.pot).

   166

    This process updates all line location number in the language-specific

   167

    .po files and may also create new entries for translation (or comment out

   168

    some).  The .po file is not changed, instead a new file is created with

   169

    the .new extension appended to the name of the .po file.

   170

   171

    By default the function does not display what it is doing.  Set the

   172

    verbose argument to 1 to force it to print its commands.

   173

    """

   174

   175

    if applicationDomain is None:

   176

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   177

    else:

   178

        applicationName = applicationDomain

   179

    currentDir = os.getcwd()

   180

    os.chdir(applicationDirectoryPath)

   181

    filelist = 'app.fil'

   182

    if not os.path.exists(filelist):

   183

        raise IOError(2, 'No module file: ' % filelist)

   184

   185

    fileout = 'messages.pot'

   186

    # Steps:

   187

    #  Use xgettext to parse all application modules

   188

    #  The following switches are used:

   189

    #

   190

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   191

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   192

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   193

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   194

    verbosePrint(verbose, cmd)

   195

    os.system(cmd)

   196

   197

    XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   198

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   199

   200

    XML_TC6_STRING_MODEL = re.compile("<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   201

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')

   202

   203

    # generate messages.po

   204

    cmd = 'msginit --no-wrap --no-translator -i %s -l en_US.UTF-8 -o messages.po' % (fileout)

   205

    verbosePrint(verbose, cmd)

   206

    os.system(cmd)

   207

   208

    languageDict = getlanguageDict()

   209

   210

    for langCode in languageDict.keys():

   211

        if langCode == 'en':

   212

            pass

   213

        else:

   214

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   215

            if os.path.exists(langPOfileName):

   216

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   217

                verbosePrint(verbose, cmd)

   218

                os.system(cmd)

   219

    os.chdir(currentDir)

   220

   221

   222

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0):

   223

    """Concatenate one or several PO files with the application domain files.

   224

    """

   225

   226

    if applicationDomain is None:

   227

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   228

    else:

   229

        applicationName = applicationDomain

   230

    currentDir = os.getcwd()

   231

    os.chdir(applicationDirectoryPath)

   232

   233

    languageDict = getlanguageDict()

   234

   235

    for langCode in languageDict.keys():

   236

        if langCode == 'en':

   237

            pass

   238

        else:

   239

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   240

            if os.path.exists(langPOfileName):

   241

                fileList = ''

   242

                for fileName in listOf_extraPo:

   243

                    fileList += ("%s_%s.po " % (fileName, langCode))

   244

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   245

                verbosePrint(verbose, cmd)

   246

                os.system(cmd)

   247

                if targetDir is None:

   248

                    pass

   249

                else:

   250

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   251

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir, applicationName, applicationName, langCode)

   252

                    verbosePrint(verbose, cmd)

   253

                    os.system(cmd)

   254

    os.chdir(currentDir)

   255

   256

   257

def makeMO(applicationDirectoryPath, targetDir='./locale', applicationDomain=None, verbose=0, forceEnglish=0):

   258

    """Compile the Portable Object files into the Machine Object stored in the right location.

   259

   260

    makeMO converts all translated language-specific PO files located inside

   261

    the  application directory into the binary .MO files stored inside the

   262

    LC_MESSAGES sub-directory for the found locale files.

   263

   264

    makeMO searches for all files that have a name of the form 'app_xx.po'

   265

    inside the application directory specified by the first argument.  The

   266

    'app' is the application domain name (that can be specified by the

   267

    applicationDomain argument or is taken from the directory name). The 'xx'

   268

    corresponds to one of the ISO 639 two-letter language codes.

   269

   270

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   271

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   272

    code.

   273

    """

   274

   275

    if targetDir is None:

   276

        targetDir = './locale'

   277

   278

    verbosePrint(verbose, "Target directory for .mo files is: %s" % targetDir)

   279

   280

    if applicationDomain is None:

   281

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   282

    else:

   283

        applicationName = applicationDomain

   284

    currentDir = os.getcwd()

   285

    os.chdir(applicationDirectoryPath)

   286

   287

    languageDict = getlanguageDict()

   288

   289

    for langCode in languageDict.keys():

   290

        if (langCode == 'en') and (forceEnglish == 0):

   291

            pass

   292

        else:

   293

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   294

            if os.path.exists(langPOfileName):

   295

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   296

                if not os.path.exists(mo_targetDir):

   297

                    mkdir(mo_targetDir)

   298

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir, applicationName, applicationName, langCode)

   299

                verbosePrint(verbose, cmd)

   300

                os.system(cmd)

   301

    os.chdir(currentDir)

   302

   303

   304

def printUsage(errorMsg=None):

   305

    """Displays how to use this script from the command line."""

   306

    print("""

   307

    ##################################################################################

   308

    #   mki18n :   Make internationalization files.                                  #

   309

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   310

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   311

    #              Supports C,C++,Python source files.                               #

   312

    #                                                                                #

   313

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   314

    #                                                                                #

   315

    #   Options:                                                                     #

   316

    #     -e               : When -m is used, forces English .mo file creation       #

   317

    #     -h               : prints this help                                        #

   318

    #     -m               : make MO from existing PO files                          #

   319

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   320

    #                        file. Creates a dom_xx.po.new for every existing        #

   321

    #                        language specific .po file. ('xx' stands for the ISO639 #

   322

    #                        two-letter language code and 'dom' stands for the       #

   323

    #                        application domain name).  mki18n requires that you     #

   324

    #                        write a 'app.fil' file  which contains the list of all  #

   325

    #                        source code to parse.                                   #

   326

    #     -v               : verbose (prints comments while running)                 #

   327

    #     --domain=appName : specifies the application domain name.  By default      #

   328

    #                        the directory name is used.                             #

   329

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   330

    #                      If not specified, the target is './locale'                #

   331

    #                                                                                #

   332

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   333

    #   specify the path of the target application.  If you leave it out mki18n      #

   334

    #   will use the current directory as the application main directory.            #

   335

    #                                                                                #

   336

    ##################################################################################""")

   337

    if errorMsg:

   338

        print("\n   ERROR: %s" % errorMsg)

   339

   340

   341

def fileBaseOf(filename, withPath=0):

   342

    """fileBaseOf(filename,withPath) ---> string

   343

   344

    Return base name of filename.  The returned string never includes the extension.

   345

    Use os.path.basename() to return the basename with the extension.  The

   346

    second argument is optional.  If specified and if set to 'true' (non zero)

   347

    the string returned contains the full path of the file name.  Otherwise the

   348

    path is excluded.

   349

   350

    [Example]

   351

    >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   352

    >>> fileBaseOf(fn)

   353

    'test'

   354

    >>> fileBaseOf(fn)

   355

    'test'

   356

    >>> fileBaseOf(fn,1)

   357

    'd:/dev/telepath/tvapp/code/test'

   358

    >>> fileBaseOf(fn,0)

   359

    'test'

   360

    >>> fn = 'abcdef'

   361

    >>> fileBaseOf(fn)

   362

    'abcdef'

   363

    >>> fileBaseOf(fn,1)

   364

    'abcdef'

   365

    >>> fn = "abcdef."

   366

    >>> fileBaseOf(fn)

   367

    'abcdef'

   368

    >>> fileBaseOf(fn,1)

   369

    'abcdef'

   370

    """

   371

    pos = filename.rfind('.')

   372

    if pos > 0:

   373

        filename = filename[:pos]

   374

    if withPath:

   375

        return filename

   376

    else:

   377

        return os.path.basename(filename)

   378

   379

   380

def mkdir(directory):

   381

    """Create a directory (and possibly the entire tree).

   382

   383

    The os.mkdir() will fail to create a directory if one of the

   384

    directory in the specified path does not exist.  mkdir()

   385

    solves this problem.  It creates every intermediate directory

   386

    required to create the final path. Under Unix, the function

   387

    only supports forward slash separator, but under Windows and MacOS

   388

    the function supports the forward slash and the OS separator (backslash

   389

    under windows).

   390

    """

   391

   392

    # translate the path separators

   393

    directory = unixpath(directory)

   394

    # build a list of all directory elements

   395

    aList = filter(lambda x: len(x) > 0, directory.split('/'))

   396

    theLen = len(aList)

   397

    # if the first element is a Windows-style disk drive

   398

    # concatenate it with the first directory

   399

    if aList[0].endswith(':'):

   400

        if theLen > 1:

   401

            aList[1] = aList[0] + '/' + aList[1]

   402

            del aList[0]

   403

            theLen -= 1

   404

    # if the original directory starts at root,

   405

    # make sure the first element of the list

   406

    # starts at root too

   407

    if directory[0] == '/':

   408

        aList[0] = '/' + aList[0]

   409

    # Now iterate through the list, check if the

   410

    # directory exists and if not create it

   411

    theDir = ''

   412

    for i in range(theLen):

   413

        theDir += aList[i]

   414

        if not os.path.exists(theDir):

   415

            os.mkdir(theDir)

   416

        theDir += '/'

   417

   418

   419

def unixpath(thePath):

   420

    r"""Return a path name that contains Unix separator.

   421

   422

    [Example]

   423

    >>> unixpath(r"d:\test")

   424

    'd:/test'

   425

    >>> unixpath("d:/test/file.txt")

   426

    'd:/test/file.txt'

   427

    >>>

   428

    """

   429

    thePath = os.path.normpath(thePath)

   430

    if os.sep == '/':

   431

        return thePath

   432

    else:

   433

        return thePath.replace(os.sep, '/')

   434

   435

   436

# -----------------------------------------------------------------------------

   437

   438

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   439

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   440

#

   441

if __name__ == "__main__":

   444

    argc = len(sys.argv)

   445

    if argc == 1:

   446

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   447

        sys.exit(1)

   448

    # If there is some arguments, parse the command line

   449

    validOptions = "ehmpv"

   450

    validLongOptions = ['domain=', 'moTarget=']

   451

    option = {}

   452

    option['forceEnglish'] = 0

   453

    option['mo'] = 0

   454

    option['po'] = 0

   455

    option['verbose'] = 0

   456

    option['domain'] = None

   457

    option['moTarget'] = None

   458

    optionKey = {

   459

        '-e':         'forceEnglish',

   460

        '-m':         'mo',

   461

        '-p':         'po',

   462

        '-v':         'verbose',

   463

        '--domain':   'domain',

   464

        '--moTarget': 'moTarget',

   465

    }

   466

    exit_code = 1

   467

    try:

   468

        optionList, pargs = getopt.getopt(sys.argv[1:], validOptions, validLongOptions)

   469

    except getopt.GetoptError, e:

   470

        printUsage(e[0])

   471

        sys.exit(1)

   472

    for (opt, val) in optionList:

   473

        if opt == '-h':

   474

            printUsage()

   475

            sys.exit(0)

   476

        option[optionKey[opt]] = 1 if val == '' else val

   477

    if len(pargs) == 0:

   478

        appDirPath = os.getcwd()

   479

        if option['verbose']:

   480

            print("No project directory given. Using current one:  %s" % appDirPath)

   481

    elif len(pargs) == 1:

   482

        appDirPath = pargs[0]

   483

    else:

   484

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   485

        sys.exit(1)

   486

    if option['domain'] is None:

   487

        # If no domain specified, use the name of the target directory

   488

        option['domain'] = fileBaseOf(appDirPath)

   489

    if option['verbose']:

   490

        print("Application domain used is: '%s'" % option['domain'])