1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

# 

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

# 

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

# 

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

# 

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

# 

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

# 

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

# 

    17

#   Update history:

    18

# 

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

# 

    31

#   RCS $Log: $

    32

# 

    33

# 

    34

# -----------------------------------------------------------------------------

    35

"""                                

    36

mki18n allows you to internationalize your software.  You can use it to 

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see 

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files: 

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_ 

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_ 

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip           

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip 

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

# 

    81

import os

    82

import sys

    83

import wx

    84

import re

    85

    86

# -----------------------------------------------------------------------------

    87

# Global variables

    88

# ----------------

    89

#

    90

    91

__author__ = "Pierre Rouleau"

    92

__version__= "$Revision: 1.5 $"

    93

    94

# -----------------------------------------------------------------------------

    95

    96

def getlanguageDict():

    97

    languageDict = {}

   104

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   105

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   106

        if i:

   107

            languageDict[i.CanonicalName] = i.Description

   108

   109

    return languageDict

   110

   111

XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   112

   113

# -----------------------------------------------------------------------------

   114

# m a k e P O ( )         -- Build the Portable Object file for the application --

   115

# ^^^^^^^^^^^^^^^

   116

#

   117

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0) :

   118

    """Build the Portable Object Template file for the application.

   119

   120

    makePO builds the .pot file for the application stored inside 

   121

    a specified directory by running xgettext for all application source 

   122

    files.  It finds the name of all files by looking for a file called 'app.fil'. 

   123

    If this file does not exists, makePo raises an IOError exception.

   124

    By default the application domain (the application

   125

    name) is the same as the directory name but it can be overridden by the

   126

    'applicationDomain' argument.

   127

   128

    makePO always creates a new file called messages.pot.  If it finds files 

   129

    of the form app_xx.po where 'app' is the application name and 'xx' is one 

   130

    of the ISO 639 two-letter language codes, makePO resynchronizes those 

   131

    files with the latest extracted strings (now contained in messages.pot). 

   132

    This process updates all line location number in the language-specific

   133

    .po files and may also create new entries for translation (or comment out 

   134

    some).  The .po file is not changed, instead a new file is created with 

   135

    the .new extension appended to the name of the .po file.

   136

   137

    By default the function does not display what it is doing.  Set the 

   138

    verbose argument to 1 to force it to print its commands.

   139

    """

   140

   141

    if applicationDomain is None:

   142

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   143

    else:

   144

        applicationName = applicationDomain

   145

    currentDir = os.getcwd()

   146

    os.chdir(applicationDirectoryPath)                    

   147

    if not os.path.exists('app.fil'):

   148

        raise IOError(2,'No module file: app.fil')

   149

   150

    # Steps:                                  

   151

    #  Use xgettext to parse all application modules

   152

    #  The following switches are used:

   153

    #  

   154

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   155

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   156

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   157

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=app.fil --output=messages.pot'

   158

    if verbose: print cmd

   159

    os.system(cmd)                                                

   160

   161

    appfil_file = open("app.fil", 'r')

   162

    messages_file = open("messages.pot", 'r')

   163

    messages = messages_file.read()

   164

    messages_file.close()

   165

    messages_file = open("messages.pot", 'a')

   166

    messages_file.write("""

   167

#: Extra XSD strings

   168

""")

   169

    words_found = {}

   170

    for filepath in appfil_file.xreadlines():

   171

        code_file = open(filepath.strip(), 'r')

   172

        for match in XSD_STRING_MODEL.finditer(code_file.read()):

   173

            word = match.group(1)

   174

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   175

                words_found[word] = True

   176

                messages_file.write("""

   177

msgid "%s"

   178

msgstr ""

   179

"""%word)

   180

        code_file.close()

   181

    messages_file.close()

   182

    appfil_file.close()

   183

   184

    languageDict = getlanguageDict()

   185

   186

    for langCode in languageDict.keys():

   187

        if langCode == 'en':

   188

            pass

   189

        else:

   190

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   191

            if os.path.exists(langPOfileName):

   192

                cmd = 'msgmerge -s --no-wrap "%s" messages.pot > "%s.new"' % (langPOfileName, langPOfileName)

   193

                if verbose: print cmd

   194

                os.system(cmd)

   195

    os.chdir(currentDir)

   196

   197

# -----------------------------------------------------------------------------

   198

# c a t P O ( )         -- Concatenate one or several PO files with the application domain files. --

   199

# ^^^^^^^^^^^^^

   200

#

   201

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0) :

   202

    """Concatenate one or several PO files with the application domain files.

   203

    """

   204

   205

    if applicationDomain is None:

   206

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   207

    else:

   208

        applicationName = applicationDomain

   209

    currentDir = os.getcwd()

   210

    os.chdir(applicationDirectoryPath)

   211

   212

    languageDict = getlanguageDict()

   213

   214

    for langCode in languageDict.keys():

   215

        if langCode == 'en':

   216

            pass

   217

        else:

   218

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   219

            if os.path.exists(langPOfileName):

   220

                fileList = ''

   221

                for fileName in listOf_extraPo:

   222

                    fileList += ("%s_%s.po " % (fileName,langCode))

   223

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   224

                if verbose: print cmd

   225

                os.system(cmd)

   226

                if targetDir is None:

   227

                    pass

   228

                else:

   229

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   230

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir,applicationName,applicationName,langCode)

   231

                    if verbose: print cmd

   232

                    os.system(cmd)

   233

    os.chdir(currentDir)

   234

   235

# -----------------------------------------------------------------------------

   236

# m a k e M O ( )         -- Compile the Portable Object files into the Machine Object stored in the right location. --

   237

# ^^^^^^^^^^^^^^^

   238

# 

   239

def makeMO(applicationDirectoryPath,targetDir='./locale',applicationDomain=None, verbose=0, forceEnglish=0) :

   240

    """Compile the Portable Object files into the Machine Object stored in the right location.

   241

   242

    makeMO converts all translated language-specific PO files located inside 

   243

    the  application directory into the binary .MO files stored inside the 

   244

    LC_MESSAGES sub-directory for the found locale files.

   245

   246

    makeMO searches for all files that have a name of the form 'app_xx.po' 

   247

    inside the application directory specified by the first argument.  The 

   248

    'app' is the application domain name (that can be specified by the 

   249

    applicationDomain argument or is taken from the directory name). The 'xx' 

   250

    corresponds to one of the ISO 639 two-letter language codes.

   251

   252

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   253

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   254

    code.

   255

    """

   256

    if targetDir is None:

   257

        targetDir = './locale'

   258

    if verbose:

   259

        print "Target directory for .mo files is: %s" % targetDir

   260

   261

    if applicationDomain is None:

   262

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   263

    else:

   264

        applicationName = applicationDomain

   265

    currentDir = os.getcwd()

   266

    os.chdir(applicationDirectoryPath)

   267

   268

    languageDict = getlanguageDict()

   269

   270

    for langCode in languageDict.keys():

   271

        if (langCode == 'en') and (forceEnglish==0):

   272

            pass

   273

        else:

   274

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   275

            if os.path.exists(langPOfileName):

   276

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   277

                if not os.path.exists(mo_targetDir):

   278

                    mkdir(mo_targetDir)

   279

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir,applicationName,applicationName,langCode)

   280

                if verbose: print cmd

   281

                os.system(cmd)

   282

    os.chdir(currentDir)

   283

   284

# -----------------------------------------------------------------------------

   285

# p r i n t U s a g e         -- Displays how to use this script from the command line --

   286

# ^^^^^^^^^^^^^^^^^^^

   287

#

   288

def printUsage(errorMsg=None) :

   289

    """Displays how to use this script from the command line."""

   290

    print """

   291

    ##################################################################################

   292

    #   mki18n :   Make internationalization files.                                  #

   293

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   294

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   295

    #              Supports C,C++,Python source files.                               #

   296

    #                                                                                #

   297

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   298

    #                                                                                #

   299

    #   Options:                                                                     #

   300

    #     -e               : When -m is used, forces English .mo file creation       #

   301

    #     -h               : prints this help                                        #

   302

    #     -m               : make MO from existing PO files                          #

   303

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   304

    #                        file. Creates a dom_xx.po.new for every existing        #

   305

    #                        language specific .po file. ('xx' stands for the ISO639 #

   306

    #                        two-letter language code and 'dom' stands for the       #

   307

    #                        application domain name).  mki18n requires that you     #

   308

    #                        write a 'app.fil' file  which contains the list of all  #

   309

    #                        source code to parse.                                   #

   310

    #     -v               : verbose (prints comments while running)                 #

   311

    #     --domain=appName : specifies the application domain name.  By default      #

   312

    #                        the directory name is used.                             #

   313

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   314

    #                      If not specified, the target is './locale'                #

   315

    #                                                                                #

   316

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   317

    #   specify the path of the target application.  If you leave it out mki18n      #

   318

    #   will use the current directory as the application main directory.            #        

   319

    #                                                                                #

   320

    ##################################################################################"""

   321

    if errorMsg:

   322

        print "\n   ERROR: %s" % errorMsg

   323

   324

# -----------------------------------------------------------------------------

   325

# f i l e B a s e O f ( )         -- Return base name of filename --

   326

# ^^^^^^^^^^^^^^^^^^^^^^^

   327

# 

   328

def fileBaseOf(filename,withPath=0) :

   329

   """fileBaseOf(filename,withPath) ---> string

   330

   331

   Return base name of filename.  The returned string never includes the extension.

   332

   Use os.path.basename() to return the basename with the extension.  The 

   333

   second argument is optional.  If specified and if set to 'true' (non zero) 

   334

   the string returned contains the full path of the file name.  Otherwise the 

   335

   path is excluded.

   336

   337

   [Example]

   338

   >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   339

   >>> fileBaseOf(fn)

   340

   'test'

   341

   >>> fileBaseOf(fn)

   342

   'test'

   343

   >>> fileBaseOf(fn,1)

   344

   'd:/dev/telepath/tvapp/code/test'

   345

   >>> fileBaseOf(fn,0)

   346

   'test'

   347

   >>> fn = 'abcdef'

   348

   >>> fileBaseOf(fn)

   349

   'abcdef'

   350

   >>> fileBaseOf(fn,1)

   351

   'abcdef'

   352

   >>> fn = "abcdef."

   353

   >>> fileBaseOf(fn)

   354

   'abcdef'

   355

   >>> fileBaseOf(fn,1)

   356

   'abcdef'

   357

   """            

   358

   pos = filename.rfind('.')             

   359

   if pos > 0:

   360

      filename = filename[:pos]

   361

   if withPath:

   362

      return filename

   363

   else:

   364

      return os.path.basename(filename)

   365

# -----------------------------------------------------------------------------

   366

# m k d i r ( )         -- Create a directory (and possibly the entire tree) --

   367

# ^^^^^^^^^^^^^

   368

# 

   369

def mkdir(directory) :

   370

   """Create a directory (and possibly the entire tree).

   371

   372

   The os.mkdir() will fail to create a directory if one of the

   373

   directory in the specified path does not exist.  mkdir()

   374

   solves this problem.  It creates every intermediate directory

   375

   required to create the final path. Under Unix, the function 

   376

   only supports forward slash separator, but under Windows and MacOS

   377

   the function supports the forward slash and the OS separator (backslash

   378

   under windows).

   379

   """ 

   380

   381

   # translate the path separators

   382

   directory = unixpath(directory)

   383

   # build a list of all directory elements

   384

   aList = filter(lambda x: len(x)>0, directory.split('/'))

   385

   theLen = len(aList)                     

   386

   # if the first element is a Windows-style disk drive

   387

   # concatenate it with the first directory

   388

   if aList[0].endswith(':'):

   389

      if theLen > 1:

   390

         aList[1] = aList[0] + '/' + aList[1]

   391

         del aList[0]      

   392

         theLen -= 1         

   393

   # if the original directory starts at root,

   394

   # make sure the first element of the list 

   395

   # starts at root too

   396

   if directory[0] == '/':     

   397

      aList[0] = '/' + aList[0]

   398

   # Now iterate through the list, check if the 

   399

   # directory exists and if not create it

   400

   theDir = ''

   401

   for i in range(theLen):

   402

      theDir += aList[i]

   403

      if not os.path.exists(theDir):

   404

         os.mkdir(theDir)

   405

      theDir += '/'   

   406

   407

# -----------------------------------------------------------------------------

   408

# u n i x p a t h ( )         -- Return a path name that contains Unix separator. --

   409

# ^^^^^^^^^^^^^^^^^^^

   410

# 

   411

def unixpath(thePath) :

   412

   r"""Return a path name that contains Unix separator.

   413

   414

   [Example]

   415

   >>> unixpath(r"d:\test")

   416

   'd:/test'

   417

   >>> unixpath("d:/test/file.txt")

   418

   'd:/test/file.txt'

   419

   >>> 

   420

   """

   421

   thePath = os.path.normpath(thePath)

   422

   if os.sep == '/':

   423

      return thePath

   424

   else:

   425

      return thePath.replace(os.sep,'/')

   426

   427

# ----------------------------------------------------------------------------- 

   428

   429

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   430

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   431

# 

   432

if __name__ == "__main__":

   433

    import getopt     # command line parsing

   434

    argc = len(sys.argv)

   435

    if argc == 1:

   436

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   437

        sys.exit(1)

   438

    # If there is some arguments, parse the command line

   439

    validOptions     = "ehmpv"

   440

    validLongOptions = ['domain=', 'moTarget=']             

   441

    option = {}

   442

    option['forceEnglish'] = 0

   443

    option['mo'] = 0

   444

    option['po'] = 0        

   445

    option['verbose'] = 0

   446

    option['domain'] = None

   447

    option['moTarget'] = None

   448

    try:

   449

        optionList,pargs = getopt.getopt(sys.argv[1:],validOptions,validLongOptions)

   450

    except getopt.GetoptError, e:

   451

        printUsage(e[0])

   452

        sys.exit(1)       

   453

    for (opt,val) in optionList:

   454

        if  (opt == '-h'):    

   455

            printUsage()

   456

            sys.exit(0) 

   457

        elif (opt == '-e'):         option['forceEnglish'] = 1

   458

        elif (opt == '-m'):         option['mo'] = 1

   459

        elif (opt == '-p'):         option['po'] = 1

   460

        elif (opt == '-v'):         option['verbose'] = 1

   461

        elif (opt == '--domain'):   option['domain'] = val

   462

        elif (opt == '--moTarget'): option['moTarget'] = val

   463

    if len(pargs) == 0:

   464

        appDirPath = os.getcwd()

   465

        if option['verbose']:

   466

            print "No project directory given. Using current one:  %s" % appDirPath

   467

    elif len(pargs) == 1:

   468

        appDirPath = pargs[0]

   469

    else:

   470

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   471

        sys.exit(1)

   472

    if option['domain'] is None:

   473

        # If no domain specified, use the name of the target directory

   474

        option['domain'] = fileBaseOf(appDirPath)

   475

    if option['verbose']:

   476

        print "Application domain used is: '%s'" % option['domain']

   477

    if option['po']:

   478

        try:

   479

            makePO(appDirPath,option['domain'],option['verbose'])

   480

        except IOError, e:

   481

            printUsage(e[1] + '\n   You must write a file app.fil that contains the list of all files to parse.')

   482

    if option['mo']:

   483

        makeMO(appDirPath,option['moTarget'],option['domain'],option['verbose'],option['forceEnglish'])

   484

    sys.exit(1)            

   485

   486

   487

# -----------------------------------------------------------------------------