1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

# 

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

# 

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

# 

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

# 

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

# 

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

# 

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

# 

    17

#   Update history:

    18

# 

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

# 

    31

#   RCS $Log: $

    32

# 

    33

# 

    34

# -----------------------------------------------------------------------------

    35

"""                                

    36

mki18n allows you to internationalize your software.  You can use it to 

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see 

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files: 

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_ 

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_ 

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip           

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip 

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

# 

    81

import os

    82

import sys

    83

import wx

    84

import re

    85

    86

# -----------------------------------------------------------------------------

    87

# Global variables

    88

# ----------------

    89

#

    90

    91

__author__ = "Pierre Rouleau"

    92

__version__= "$Revision: 1.5 $"

    93

    94

# -----------------------------------------------------------------------------

    95

    96

def getlanguageDict():

    97

    languageDict = {}

    98

    99

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   100

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   101

        if i:

   102

            languageDict[i.CanonicalName] = i.Description

   103

   104

    return languageDict

   105

   106

XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   107

   108

# -----------------------------------------------------------------------------

   109

# m a k e P O ( )         -- Build the Portable Object file for the application --

   110

# ^^^^^^^^^^^^^^^

   111

#

   112

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0) :

   113

    """Build the Portable Object Template file for the application.

   114

   115

    makePO builds the .pot file for the application stored inside 

   116

    a specified directory by running xgettext for all application source 

   117

    files.  It finds the name of all files by looking for a file called 'app.fil'. 

   118

    If this file does not exists, makePo raises an IOError exception.

   119

    By default the application domain (the application

   120

    name) is the same as the directory name but it can be overridden by the

   121

    'applicationDomain' argument.

   122

   123

    makePO always creates a new file called messages.pot.  If it finds files 

   124

    of the form app_xx.po where 'app' is the application name and 'xx' is one 

   125

    of the ISO 639 two-letter language codes, makePO resynchronizes those 

   126

    files with the latest extracted strings (now contained in messages.pot). 

   127

    This process updates all line location number in the language-specific

   128

    .po files and may also create new entries for translation (or comment out 

   129

    some).  The .po file is not changed, instead a new file is created with 

   130

    the .new extension appended to the name of the .po file.

   131

   132

    By default the function does not display what it is doing.  Set the 

   133

    verbose argument to 1 to force it to print its commands.

   134

    """

   135

   136

    if applicationDomain is None:

   137

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   138

    else:

   139

        applicationName = applicationDomain

   140

    currentDir = os.getcwd()

   141

    os.chdir(applicationDirectoryPath)                    

   142

    if not os.path.exists('app.fil'):

   143

        raise IOError(2,'No module file: app.fil')

   144

   145

    # Steps:                                  

   146

    #  Use xgettext to parse all application modules

   147

    #  The following switches are used:

   148

    #  

   149

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   150

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   151

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   152

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=app.fil --output=messages.pot'

   153

    if verbose: print cmd

   154

    os.system(cmd)                                                

   155

   156

    appfil_file = open("app.fil", 'r')

   157

    messages_file = open("messages.pot", 'r')

   158

    messages = messages_file.read()

   159

    messages_file.close()

   160

    messages_file = open("messages.pot", 'a')

   161

    messages_file.write("""

   162

#: Extra XSD strings

   163

""")

   164

    words_found = {}

   165

    for filepath in appfil_file.xreadlines():

   166

        code_file = open(filepath.strip(), 'r')

   167

        for match in XSD_STRING_MODEL.finditer(code_file.read()):

   168

            word = match.group(1)

   169

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   170

                words_found[word] = True

   171

                messages_file.write("""

   172

msgid "%s"

   173

msgstr ""

   174

"""%word)

   175

        code_file.close()

   176

    messages_file.close()

   177

    appfil_file.close()

   178

   179

    languageDict = getlanguageDict()

   180

   181

    for langCode in languageDict.keys():

   182

        if langCode == 'en':

   183

            pass

   184

        else:

   185

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   186

            if os.path.exists(langPOfileName):

   187

                cmd = 'msgmerge -s --no-wrap "%s" messages.pot > "%s.new"' % (langPOfileName, langPOfileName)

   188

                if verbose: print cmd

   189

                os.system(cmd)

   190

    os.chdir(currentDir)

   191

   192

# -----------------------------------------------------------------------------

   193

# c a t P O ( )         -- Concatenate one or several PO files with the application domain files. --

   194

# ^^^^^^^^^^^^^

   195

#

   196

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0) :

   197

    """Concatenate one or several PO files with the application domain files.

   198

    """

   199

   200

    if applicationDomain is None:

   201

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   202

    else:

   203

        applicationName = applicationDomain

   204

    currentDir = os.getcwd()

   205

    os.chdir(applicationDirectoryPath)

   206

   207

    languageDict = getlanguageDict()

   208

   209

    for langCode in languageDict.keys():

   210

        if langCode == 'en':

   211

            pass

   212

        else:

   213

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   214

            if os.path.exists(langPOfileName):

   215

                fileList = ''

   216

                for fileName in listOf_extraPo:

   217

                    fileList += ("%s_%s.po " % (fileName,langCode))

   218

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   219

                if verbose: print cmd

   220

                os.system(cmd)

   221

                if targetDir is None:

   222

                    pass

   223

                else:

   224

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   225

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir,applicationName,applicationName,langCode)

   226

                    if verbose: print cmd

   227

                    os.system(cmd)

   228

    os.chdir(currentDir)

   229

   230

# -----------------------------------------------------------------------------

   231

# m a k e M O ( )         -- Compile the Portable Object files into the Machine Object stored in the right location. --

   232

# ^^^^^^^^^^^^^^^

   233

# 

   234

def makeMO(applicationDirectoryPath,targetDir='./locale',applicationDomain=None, verbose=0, forceEnglish=0) :

   235

    """Compile the Portable Object files into the Machine Object stored in the right location.

   236

   237

    makeMO converts all translated language-specific PO files located inside 

   238

    the  application directory into the binary .MO files stored inside the 

   239

    LC_MESSAGES sub-directory for the found locale files.

   240

   241

    makeMO searches for all files that have a name of the form 'app_xx.po' 

   242

    inside the application directory specified by the first argument.  The 

   243

    'app' is the application domain name (that can be specified by the 

   244

    applicationDomain argument or is taken from the directory name). The 'xx' 

   245

    corresponds to one of the ISO 639 two-letter language codes.

   246

   247

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   248

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   249

    code.

   250

    """

   251

    if targetDir is None:

   252

        targetDir = './locale'

   253

    if verbose:

   254

        print "Target directory for .mo files is: %s" % targetDir

   255

   256

    if applicationDomain is None:

   257

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   258

    else:

   259

        applicationName = applicationDomain

   260

    currentDir = os.getcwd()

   261

    os.chdir(applicationDirectoryPath)

   262

   263

    languageDict = getlanguageDict()

   264

   265

    for langCode in languageDict.keys():

   266

        if (langCode == 'en') and (forceEnglish==0):

   267

            pass

   268

        else:

   269

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   270

            if os.path.exists(langPOfileName):

   271

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   272

                if not os.path.exists(mo_targetDir):

   273

                    mkdir(mo_targetDir)

   274

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir,applicationName,applicationName,langCode)

   275

                if verbose: print cmd

   276

                os.system(cmd)

   277

    os.chdir(currentDir)

   278

   279

# -----------------------------------------------------------------------------

   280

# p r i n t U s a g e         -- Displays how to use this script from the command line --

   281

# ^^^^^^^^^^^^^^^^^^^

   282

#

   283

def printUsage(errorMsg=None) :

   284

    """Displays how to use this script from the command line."""

   285

    print """

   286

    ##################################################################################

   287

    #   mki18n :   Make internationalization files.                                  #

   288

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   289

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   290

    #              Supports C,C++,Python source files.                               #

   291

    #                                                                                #

   292

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   293

    #                                                                                #

   294

    #   Options:                                                                     #

   295

    #     -e               : When -m is used, forces English .mo file creation       #

   296

    #     -h               : prints this help                                        #

   297

    #     -m               : make MO from existing PO files                          #

   298

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   299

    #                        file. Creates a dom_xx.po.new for every existing        #

   300

    #                        language specific .po file. ('xx' stands for the ISO639 #

   301

    #                        two-letter language code and 'dom' stands for the       #

   302

    #                        application domain name).  mki18n requires that you     #

   303

    #                        write a 'app.fil' file  which contains the list of all  #

   304

    #                        source code to parse.                                   #

   305

    #     -v               : verbose (prints comments while running)                 #

   306

    #     --domain=appName : specifies the application domain name.  By default      #

   307

    #                        the directory name is used.                             #

   308

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   309

    #                      If not specified, the target is './locale'                #

   310

    #                                                                                #

   311

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   312

    #   specify the path of the target application.  If you leave it out mki18n      #

   313

    #   will use the current directory as the application main directory.            #        

   314

    #                                                                                #

   315

    ##################################################################################"""

   316

    if errorMsg:

   317

        print "\n   ERROR: %s" % errorMsg

   318

   319

# -----------------------------------------------------------------------------

   320

# f i l e B a s e O f ( )         -- Return base name of filename --

   321

# ^^^^^^^^^^^^^^^^^^^^^^^

   322

# 

   323

def fileBaseOf(filename,withPath=0) :

   324

   """fileBaseOf(filename,withPath) ---> string

   325

   326

   Return base name of filename.  The returned string never includes the extension.

   327

   Use os.path.basename() to return the basename with the extension.  The 

   328

   second argument is optional.  If specified and if set to 'true' (non zero) 

   329

   the string returned contains the full path of the file name.  Otherwise the 

   330

   path is excluded.

   331

   332

   [Example]

   333

   >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   334

   >>> fileBaseOf(fn)

   335

   'test'

   336

   >>> fileBaseOf(fn)

   337

   'test'

   338

   >>> fileBaseOf(fn,1)

   339

   'd:/dev/telepath/tvapp/code/test'

   340

   >>> fileBaseOf(fn,0)

   341

   'test'

   342

   >>> fn = 'abcdef'

   343

   >>> fileBaseOf(fn)

   344

   'abcdef'

   345

   >>> fileBaseOf(fn,1)

   346

   'abcdef'

   347

   >>> fn = "abcdef."

   348

   >>> fileBaseOf(fn)

   349

   'abcdef'

   350

   >>> fileBaseOf(fn,1)

   351

   'abcdef'

   352

   """            

   353

   pos = filename.rfind('.')             

   354

   if pos > 0:

   355

      filename = filename[:pos]

   356

   if withPath:

   357

      return filename

   358

   else:

   359

      return os.path.basename(filename)

   360

# -----------------------------------------------------------------------------

   361

# m k d i r ( )         -- Create a directory (and possibly the entire tree) --

   362

# ^^^^^^^^^^^^^

   363

# 

   364

def mkdir(directory) :

   365

   """Create a directory (and possibly the entire tree).

   366

   367

   The os.mkdir() will fail to create a directory if one of the

   368

   directory in the specified path does not exist.  mkdir()

   369

   solves this problem.  It creates every intermediate directory

   370

   required to create the final path. Under Unix, the function 

   371

   only supports forward slash separator, but under Windows and MacOS

   372

   the function supports the forward slash and the OS separator (backslash

   373

   under windows).

   374

   """ 

   375

   376

   # translate the path separators

   377

   directory = unixpath(directory)

   378

   # build a list of all directory elements

   379

   aList = filter(lambda x: len(x)>0, directory.split('/'))

   380

   theLen = len(aList)                     

   381

   # if the first element is a Windows-style disk drive

   382

   # concatenate it with the first directory

   383

   if aList[0].endswith(':'):

   384

      if theLen > 1:

   385

         aList[1] = aList[0] + '/' + aList[1]

   386

         del aList[0]      

   387

         theLen -= 1         

   388

   # if the original directory starts at root,

   389

   # make sure the first element of the list 

   390

   # starts at root too

   391

   if directory[0] == '/':     

   392

      aList[0] = '/' + aList[0]

   393

   # Now iterate through the list, check if the 

   394

   # directory exists and if not create it

   395

   theDir = ''

   396

   for i in range(theLen):

   397

      theDir += aList[i]

   398

      if not os.path.exists(theDir):

   399

         os.mkdir(theDir)

   400

      theDir += '/'   

   401

   402

# -----------------------------------------------------------------------------

   403

# u n i x p a t h ( )         -- Return a path name that contains Unix separator. --

   404

# ^^^^^^^^^^^^^^^^^^^

   405

# 

   406

def unixpath(thePath) :

   407

   r"""Return a path name that contains Unix separator.

   408

   409

   [Example]

   410

   >>> unixpath(r"d:\test")

   411

   'd:/test'

   412

   >>> unixpath("d:/test/file.txt")

   413

   'd:/test/file.txt'

   414

   >>> 

   415

   """

   416

   thePath = os.path.normpath(thePath)

   417

   if os.sep == '/':

   418

      return thePath

   419

   else:

   420

      return thePath.replace(os.sep,'/')

   421

   422

# ----------------------------------------------------------------------------- 

   423

   424

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   425

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   426

# 

   427

if __name__ == "__main__":

   428

    import getopt     # command line parsing

   429

    argc = len(sys.argv)

   430

    if argc == 1:

   431

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   432

        sys.exit(1)

   433

    # If there is some arguments, parse the command line

   434

    validOptions     = "ehmpv"

   435

    validLongOptions = ['domain=', 'moTarget=']             

   436

    option = {}

   437

    option['forceEnglish'] = 0

   438

    option['mo'] = 0

   439

    option['po'] = 0        

   440

    option['verbose'] = 0

   441

    option['domain'] = None

   442

    option['moTarget'] = None

   443

    try:

   444

        optionList,pargs = getopt.getopt(sys.argv[1:],validOptions,validLongOptions)

   445

    except getopt.GetoptError, e:

   446

        printUsage(e[0])

   447

        sys.exit(1)       

   448

    for (opt,val) in optionList:

   449

        if  (opt == '-h'):    

   450

            printUsage()

   451

            sys.exit(0) 

   452

        elif (opt == '-e'):         option['forceEnglish'] = 1

   453

        elif (opt == '-m'):         option['mo'] = 1

   454

        elif (opt == '-p'):         option['po'] = 1

   455

        elif (opt == '-v'):         option['verbose'] = 1

   456

        elif (opt == '--domain'):   option['domain'] = val

   457

        elif (opt == '--moTarget'): option['moTarget'] = val

   458

    if len(pargs) == 0:

   459

        appDirPath = os.getcwd()

   460

        if option['verbose']:

   461

            print "No project directory given. Using current one:  %s" % appDirPath

   462

    elif len(pargs) == 1:

   463

        appDirPath = pargs[0]

   464

    else:

   465

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   466

        sys.exit(1)

   467

    if option['domain'] is None:

   468

        # If no domain specified, use the name of the target directory

   469

        option['domain'] = fileBaseOf(appDirPath)

   470

    if option['verbose']:

   471

        print "Application domain used is: '%s'" % option['domain']

   472

    if option['po']:

   473

        try:

   474

            makePO(appDirPath,option['domain'],option['verbose'])

   475

        except IOError, e:

   476

            printUsage(e[1] + '\n   You must write a file app.fil that contains the list of all files to parse.')

   477

    if option['mo']:

   478

        makeMO(appDirPath,option['moTarget'],option['domain'],option['verbose'],option['forceEnglish'])

   479

    sys.exit(1)            

   480

   481

   482

# -----------------------------------------------------------------------------