1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

#

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

#

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

#

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

#

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

#

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

#

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

#

    17

#   Update history:

    18

#

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

#

    31

#   RCS $Log: $

    32

#

    33

#

    34

# -----------------------------------------------------------------------------

    35

"""

    36

mki18n allows you to internationalize your software.  You can use it to

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files:

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

#

    81

import os

    82

import sys

    83

import wx

    84

import re

    85

    86

# -----------------------------------------------------------------------------

    87

# Global variables

    88

# ----------------

    89

#

    90

    91

__author__ = "Pierre Rouleau"

    92

__version__ = "$Revision: 1.5 $"

    93

    94

# -----------------------------------------------------------------------------

    95

    96

    97

def getlanguageDict():

    98

    languageDict = {}

    99

   100

    if wx.VERSION >= (3, 0, 0):

   101

        app = wx.App()

   102

    else:

   103

        app = wx.PySimpleApp()

   104

   105

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   106

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   107

        if i:

   108

            languageDict[i.CanonicalName] = i.Description

   109

   110

    return languageDict

   111

   112

   113

def verbosePrint(verbose, str):

   114

    if verbose:

   115

        print str

   116

   117

   118

def processCustomFiles(filein, fileout, regexp, prefix=''):

   119

    appfil_file = open(filein, 'r')

   120

    messages_file = open(fileout, 'r')

   121

    messages = messages_file.read()

   122

    messages_file.close()

   123

    messages_file = open(fileout, 'a')

   124

    messages_file.write('\n')

   125

    messages_file.write('#: %s\n' % prefix)

   126

    messages_file.write('\n')

   127

   128

    words_found = {}

   129

    for filepath in appfil_file.xreadlines():

   130

        code_file = open(filepath.strip(), 'r')

   131

        for match in regexp.finditer(code_file.read()):

   132

            word = match.group(1)

   133

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   134

                words_found[word] = True

   135

                messages_file.write('\n')

   136

                messages_file.write("msgid \"%s\"\n" % word)

   137

                messages_file.write("msgstr \"\"\n")

   138

        code_file.close()

   139

   140

    messages_file.close()

   141

    appfil_file.close()

   142

   143

   144

# -----------------------------------------------------------------------------

   145

# m a k e P O ( )         -- Build the Portable Object file for the application --

   146

# ^^^^^^^^^^^^^^^

   147

#

   148

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0):

   149

    """Build the Portable Object Template file for the application.

   150

   151

    makePO builds the .pot file for the application stored inside

   152

    a specified directory by running xgettext for all application source

   153

    files.  It finds the name of all files by looking for a file called 'app.fil'.

   154

    If this file does not exists, makePo raises an IOError exception.

   155

    By default the application domain (the application

   156

    name) is the same as the directory name but it can be overridden by the

   157

    'applicationDomain' argument.

   158

   159

    makePO always creates a new file called messages.pot.  If it finds files

   160

    of the form app_xx.po where 'app' is the application name and 'xx' is one

   161

    of the ISO 639 two-letter language codes, makePO resynchronizes those

   162

    files with the latest extracted strings (now contained in messages.pot).

   163

    This process updates all line location number in the language-specific

   164

    .po files and may also create new entries for translation (or comment out

   165

    some).  The .po file is not changed, instead a new file is created with

   166

    the .new extension appended to the name of the .po file.

   167

   168

    By default the function does not display what it is doing.  Set the

   169

    verbose argument to 1 to force it to print its commands.

   170

    """

   171

   172

    if applicationDomain is None:

   173

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   174

    else:

   175

        applicationName = applicationDomain

   176

    currentDir = os.getcwd()

   177

    os.chdir(applicationDirectoryPath)

   178

    filelist = 'app.fil'

   179

    if not os.path.exists(filelist):

   180

        raise IOError(2, 'No module file: ' % filelist)

   181

   182

    fileout = 'messages.pot'

   183

    # Steps:

   184

    #  Use xgettext to parse all application modules

   185

    #  The following switches are used:

   186

    #

   187

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   188

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   189

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   190

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   191

    verbosePrint(verbose, cmd)

   192

    os.system(cmd)

   193

   194

    XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   195

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   196

   197

    XML_TC6_STRING_MODEL = re.compile("<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   198

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')

   199

   200

    # generate messages.po

   201

    cmd = 'msginit --no-wrap --no-translator -i %s -l en_US.UTF-8 -o messages.po' % (fileout)

   202

    verbosePrint(verbose, cmd)

   203

    os.system(cmd)

   204

   205

    languageDict = getlanguageDict()

   206

   207

    for langCode in languageDict.keys():

   208

        if langCode == 'en':

   209

            pass

   210

        else:

   211

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   212

            if os.path.exists(langPOfileName):

   213

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   214

                verbosePrint(verbose, cmd)

   215

                os.system(cmd)

   216

    os.chdir(currentDir)

   217

   218

   219

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0):

   220

    """Concatenate one or several PO files with the application domain files.

   221

    """

   222

   223

    if applicationDomain is None:

   224

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   225

    else:

   226

        applicationName = applicationDomain

   227

    currentDir = os.getcwd()

   228

    os.chdir(applicationDirectoryPath)

   229

   230

    languageDict = getlanguageDict()

   231

   232

    for langCode in languageDict.keys():

   233

        if langCode == 'en':

   234

            pass

   235

        else:

   236

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   237

            if os.path.exists(langPOfileName):

   238

                fileList = ''

   239

                for fileName in listOf_extraPo:

   240

                    fileList += ("%s_%s.po " % (fileName, langCode))

   241

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   242

                verbosePrint(verbose, cmd)

   243

                os.system(cmd)

   244

                if targetDir is None:

   245

                    pass

   246

                else:

   247

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   248

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir, applicationName, applicationName, langCode)

   249

                    verbosePrint(verbose, cmd)

   250

                    os.system(cmd)

   251

    os.chdir(currentDir)

   252

   253

   254

def makeMO(applicationDirectoryPath, targetDir='./locale', applicationDomain=None, verbose=0, forceEnglish=0):

   255

    """Compile the Portable Object files into the Machine Object stored in the right location.

   256

   257

    makeMO converts all translated language-specific PO files located inside

   258

    the  application directory into the binary .MO files stored inside the

   259

    LC_MESSAGES sub-directory for the found locale files.

   260

   261

    makeMO searches for all files that have a name of the form 'app_xx.po'

   262

    inside the application directory specified by the first argument.  The

   263

    'app' is the application domain name (that can be specified by the

   264

    applicationDomain argument or is taken from the directory name). The 'xx'

   265

    corresponds to one of the ISO 639 two-letter language codes.

   266

   267

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   268

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   269

    code.

   270

    """

   271

   272

    if targetDir is None:

   273

        targetDir = './locale'

   274

   275

    verbosePrint(verbose, "Target directory for .mo files is: %s" % targetDir)

   276

   277

    if applicationDomain is None:

   278

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   279

    else:

   280

        applicationName = applicationDomain

   281

    currentDir = os.getcwd()

   282

    os.chdir(applicationDirectoryPath)

   283

   284

    languageDict = getlanguageDict()

   285

   286

    for langCode in languageDict.keys():

   287

        if (langCode == 'en') and (forceEnglish == 0):

   288

            pass

   289

        else:

   290

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   291

            if os.path.exists(langPOfileName):

   292

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   293

                if not os.path.exists(mo_targetDir):

   294

                    mkdir(mo_targetDir)

   295

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir, applicationName, applicationName, langCode)

   296

                verbosePrint(verbose, cmd)

   297

                os.system(cmd)

   298

    os.chdir(currentDir)

   299

   300

   301

def printUsage(errorMsg=None):

   302

    """Displays how to use this script from the command line."""

   303

    print """

   304

    ##################################################################################

   305

    #   mki18n :   Make internationalization files.                                  #

   306

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   307

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   308

    #              Supports C,C++,Python source files.                               #

   309

    #                                                                                #

   310

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   311

    #                                                                                #

   312

    #   Options:                                                                     #

   313

    #     -e               : When -m is used, forces English .mo file creation       #

   314

    #     -h               : prints this help                                        #

   315

    #     -m               : make MO from existing PO files                          #

   316

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   317

    #                        file. Creates a dom_xx.po.new for every existing        #

   318

    #                        language specific .po file. ('xx' stands for the ISO639 #

   319

    #                        two-letter language code and 'dom' stands for the       #

   320

    #                        application domain name).  mki18n requires that you     #

   321

    #                        write a 'app.fil' file  which contains the list of all  #

   322

    #                        source code to parse.                                   #

   323

    #     -v               : verbose (prints comments while running)                 #

   324

    #     --domain=appName : specifies the application domain name.  By default      #

   325

    #                        the directory name is used.                             #

   326

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   327

    #                      If not specified, the target is './locale'                #

   328

    #                                                                                #

   329

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   330

    #   specify the path of the target application.  If you leave it out mki18n      #

   331

    #   will use the current directory as the application main directory.            #

   332

    #                                                                                #

   333

    ##################################################################################"""

   334

    if errorMsg:

   335

        print "\n   ERROR: %s" % errorMsg

   336

   337

   338

def fileBaseOf(filename, withPath=0):

   339

    """fileBaseOf(filename,withPath) ---> string

   340

   341

    Return base name of filename.  The returned string never includes the extension.

   342

    Use os.path.basename() to return the basename with the extension.  The

   343

    second argument is optional.  If specified and if set to 'true' (non zero)

   344

    the string returned contains the full path of the file name.  Otherwise the

   345

    path is excluded.

   346

   347

    [Example]

   348

    >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   349

    >>> fileBaseOf(fn)

   350

    'test'

   351

    >>> fileBaseOf(fn)

   352

    'test'

   353

    >>> fileBaseOf(fn,1)

   354

    'd:/dev/telepath/tvapp/code/test'

   355

    >>> fileBaseOf(fn,0)

   356

    'test'

   357

    >>> fn = 'abcdef'

   358

    >>> fileBaseOf(fn)

   359

    'abcdef'

   360

    >>> fileBaseOf(fn,1)

   361

    'abcdef'

   362

    >>> fn = "abcdef."

   363

    >>> fileBaseOf(fn)

   364

    'abcdef'

   365

    >>> fileBaseOf(fn,1)

   366

    'abcdef'

   367

    """

   368

    pos = filename.rfind('.')

   369

    if pos > 0:

   370

        filename = filename[:pos]

   371

    if withPath:

   372

        return filename

   373

    else:

   374

        return os.path.basename(filename)

   375

   376

   377

def mkdir(directory):

   378

    """Create a directory (and possibly the entire tree).

   379

   380

    The os.mkdir() will fail to create a directory if one of the

   381

    directory in the specified path does not exist.  mkdir()

   382

    solves this problem.  It creates every intermediate directory

   383

    required to create the final path. Under Unix, the function

   384

    only supports forward slash separator, but under Windows and MacOS

   385

    the function supports the forward slash and the OS separator (backslash

   386

    under windows).

   387

    """

   388

   389

    # translate the path separators

   390

    directory = unixpath(directory)

   391

    # build a list of all directory elements

   392

    aList = filter(lambda x: len(x) > 0, directory.split('/'))

   393

    theLen = len(aList)

   394

    # if the first element is a Windows-style disk drive

   395

    # concatenate it with the first directory

   396

    if aList[0].endswith(':'):

   397

        if theLen > 1:

   398

            aList[1] = aList[0] + '/' + aList[1]

   399

            del aList[0]

   400

            theLen -= 1

   401

    # if the original directory starts at root,

   402

    # make sure the first element of the list

   403

    # starts at root too

   404

    if directory[0] == '/':

   405

        aList[0] = '/' + aList[0]

   406

    # Now iterate through the list, check if the

   407

    # directory exists and if not create it

   408

    theDir = ''

   409

    for i in range(theLen):

   410

        theDir += aList[i]

   411

        if not os.path.exists(theDir):

   412

            os.mkdir(theDir)

   413

        theDir += '/'

   414

   415

   416

def unixpath(thePath):

   417

    r"""Return a path name that contains Unix separator.

   418

   419

    [Example]

   420

    >>> unixpath(r"d:\test")

   421

    'd:/test'

   422

    >>> unixpath("d:/test/file.txt")

   423

    'd:/test/file.txt'

   424

    >>>

   425

    """

   426

    thePath = os.path.normpath(thePath)

   427

    if os.sep == '/':

   428

        return thePath

   429

    else:

   430

        return thePath.replace(os.sep, '/')

   431

   432

   433

# -----------------------------------------------------------------------------

   434

   435

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   436

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   437

#

   438

if __name__ == "__main__":

   439

    import getopt     # command line parsing

   440

    argc = len(sys.argv)

   441

    if argc == 1:

   442

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   443

        sys.exit(1)

   444

    # If there is some arguments, parse the command line

   445

    validOptions = "ehmpv"

   446

    validLongOptions = ['domain=', 'moTarget=']

   447

    option = {}

   448

    option['forceEnglish'] = 0

   449

    option['mo'] = 0

   450

    option['po'] = 0

   451

    option['verbose'] = 0

   452

    option['domain'] = None

   453

    option['moTarget'] = None

   454

    optionKey = {

   455

        '-e':         'forceEnglish',

   456

        '-m':         'mo',

   457

        '-p':         'po',

   458

        '-v':         'verbose',

   459

        '--domain':   'domain',

   460

        '--moTarget': 'moTarget',

   461

    }

   462

   463

    try:

   464

        optionList, pargs = getopt.getopt(sys.argv[1:], validOptions, validLongOptions)

   465

    except getopt.GetoptError, e:

   466

        printUsage(e[0])

   467

        sys.exit(1)

   468

    for (opt, val) in optionList:

   469

        if (opt == '-h'):

   470

            printUsage()

   471

            sys.exit(0)

   472

        option[optionKey[opt]] = 1 if val == '' else val

   473

    if len(pargs) == 0:

   474

        appDirPath = os.getcwd()

   475

        if option['verbose']:

   476

            print "No project directory given. Using current one:  %s" % appDirPath

   477

    elif len(pargs) == 1:

   478

        appDirPath = pargs[0]

   479

    else:

   480

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   481

        sys.exit(1)

   482

    if option['domain'] is None:

   483

        # If no domain specified, use the name of the target directory

   484

        option['domain'] = fileBaseOf(appDirPath)

   485

    if option['verbose']:

   486

        print "Application domain used is: '%s'" % option['domain']

   487

    if option['po']:

   488

        try:

   489

            makePO(appDirPath, option['domain'], option['verbose'])

   490

        except IOError, e:

   491

            printUsage(e[1] + '\n   You must write a file app.fil that contains the list of all files to parse.')