1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

# 

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

# 

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

# 

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

# 

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

# 

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

# 

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

# 

    17

#   Update history:

    18

# 

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

# 

    31

#   RCS $Log: $

    32

# 

    33

# 

    34

# -----------------------------------------------------------------------------

    35

"""                                

    36

mki18n allows you to internationalize your software.  You can use it to 

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see 

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files: 

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_ 

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_ 

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip           

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip 

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

# 

    81

import os

    82

import sys

    83

import wx

    84

import re

    85

    86

# -----------------------------------------------------------------------------

    87

# Global variables

    88

# ----------------

    89

#

    90

    91

__author__ = "Pierre Rouleau"

    92

__version__= "$Revision: 1.5 $"

    93

    94

# -----------------------------------------------------------------------------

    95

    96

def getlanguageDict():

    97

    languageDict = {}

    98

    99

    if wx.VERSION >= (3, 0, 0):

   100

        app = wx.App()

   101

    else:

   102

        app = wx.PySimpleApp()

   103

   104

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   105

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   106

        if i:

   107

            languageDict[i.CanonicalName] = i.Description

   108

   109

    return languageDict

   110

   111

   112

   113

def processCustomFiles(filein, fileout, regexp, prefix = ''):

   114

    appfil_file = open(filein, 'r')

   115

    messages_file = open(fileout, 'r')

   116

    messages = messages_file.read()

   117

    messages_file.close()

   118

    messages_file = open(fileout, 'a')

   119

    messages_file.write('\n')

   120

    messages_file.write('#: %s\n' % prefix)

   121

    messages_file.write('\n')

   122

   123

    words_found = {}

   124

    for filepath in appfil_file.xreadlines():

   125

        code_file = open(filepath.strip(), 'r')

   126

        for match in regexp.finditer(code_file.read()):

   127

            word = match.group(1)

   128

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   129

                words_found[word] = True

   130

                messages_file.write('\n')

   131

                messages_file.write("msgid \"%s\"\n"%word)

   132

                messages_file.write("msgstr \"\"\n")

   133

        code_file.close()

   134

   135

    messages_file.close()

   136

    appfil_file.close()

   137

   138

   139

# -----------------------------------------------------------------------------

   140

# m a k e P O ( )         -- Build the Portable Object file for the application --

   141

# ^^^^^^^^^^^^^^^

   142

#

   143

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0) :

   144

    """Build the Portable Object Template file for the application.

   145

   146

    makePO builds the .pot file for the application stored inside 

   147

    a specified directory by running xgettext for all application source 

   148

    files.  It finds the name of all files by looking for a file called 'app.fil'. 

   149

    If this file does not exists, makePo raises an IOError exception.

   150

    By default the application domain (the application

   151

    name) is the same as the directory name but it can be overridden by the

   152

    'applicationDomain' argument.

   153

   154

    makePO always creates a new file called messages.pot.  If it finds files 

   155

    of the form app_xx.po where 'app' is the application name and 'xx' is one 

   156

    of the ISO 639 two-letter language codes, makePO resynchronizes those 

   157

    files with the latest extracted strings (now contained in messages.pot). 

   158

    This process updates all line location number in the language-specific

   159

    .po files and may also create new entries for translation (or comment out 

   160

    some).  The .po file is not changed, instead a new file is created with 

   161

    the .new extension appended to the name of the .po file.

   162

   163

    By default the function does not display what it is doing.  Set the 

   164

    verbose argument to 1 to force it to print its commands.

   165

    """

   166

   167

    if applicationDomain is None:

   168

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   169

    else:

   170

        applicationName = applicationDomain

   171

    currentDir = os.getcwd()

   172

    os.chdir(applicationDirectoryPath)

   173

    filelist = 'app.fil'

   174

    if not os.path.exists(filelist):

   175

        raise IOError(2,'No module file: ' % filelist)

   176

   177

    fileout = 'messages.pot'

   178

    # Steps:                                  

   179

    #  Use xgettext to parse all application modules

   180

    #  The following switches are used:

   181

    #  

   182

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   183

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   184

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   185

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   186

    if verbose: print cmd

   187

    os.system(cmd)

   188

   189

    XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   190

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   191

   192

    XML_TC6_STRING_MODEL = re.compile("<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   193

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')    

   194

   200

    languageDict = getlanguageDict()

   201

   202

    for langCode in languageDict.keys():

   203

        if langCode == 'en':

   204

            pass

   205

        else:

   206

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   207

            if os.path.exists(langPOfileName):

   208

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   209

                if verbose: print cmd

   210

                os.system(cmd)

   211

    os.chdir(currentDir)

   212

   213

# -----------------------------------------------------------------------------

   214

# c a t P O ( )         -- Concatenate one or several PO files with the application domain files. --

   215

# ^^^^^^^^^^^^^

   216

#

   217

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0) :

   218

    """Concatenate one or several PO files with the application domain files.

   219

    """

   220

   221

    if applicationDomain is None:

   222

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   223

    else:

   224

        applicationName = applicationDomain

   225

    currentDir = os.getcwd()

   226

    os.chdir(applicationDirectoryPath)

   227

   228

    languageDict = getlanguageDict()

   229

   230

    for langCode in languageDict.keys():

   231

        if langCode == 'en':

   232

            pass

   233

        else:

   234

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   235

            if os.path.exists(langPOfileName):

   236

                fileList = ''

   237

                for fileName in listOf_extraPo:

   238

                    fileList += ("%s_%s.po " % (fileName,langCode))

   239

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   240

                if verbose: print cmd

   241

                os.system(cmd)

   242

                if targetDir is None:

   243

                    pass

   244

                else:

   245

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   246

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir,applicationName,applicationName,langCode)

   247

                    if verbose: print cmd

   248

                    os.system(cmd)

   249

    os.chdir(currentDir)

   250

   251

# -----------------------------------------------------------------------------

   252

# m a k e M O ( )         -- Compile the Portable Object files into the Machine Object stored in the right location. --

   253

# ^^^^^^^^^^^^^^^

   254

# 

   255

def makeMO(applicationDirectoryPath,targetDir='./locale',applicationDomain=None, verbose=0, forceEnglish=0) :

   256

    """Compile the Portable Object files into the Machine Object stored in the right location.

   257

   258

    makeMO converts all translated language-specific PO files located inside 

   259

    the  application directory into the binary .MO files stored inside the 

   260

    LC_MESSAGES sub-directory for the found locale files.

   261

   262

    makeMO searches for all files that have a name of the form 'app_xx.po' 

   263

    inside the application directory specified by the first argument.  The 

   264

    'app' is the application domain name (that can be specified by the 

   265

    applicationDomain argument or is taken from the directory name). The 'xx' 

   266

    corresponds to one of the ISO 639 two-letter language codes.

   267

   268

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   269

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   270

    code.

   271

    """

   272

    if targetDir is None:

   273

        targetDir = './locale'

   274

    if verbose:

   275

        print "Target directory for .mo files is: %s" % targetDir

   276

   277

    if applicationDomain is None:

   278

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   279

    else:

   280

        applicationName = applicationDomain

   281

    currentDir = os.getcwd()

   282

    os.chdir(applicationDirectoryPath)

   283

   284

    languageDict = getlanguageDict()

   285

   286

    for langCode in languageDict.keys():

   287

        if (langCode == 'en') and (forceEnglish==0):

   288

            pass

   289

        else:

   290

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   291

            if os.path.exists(langPOfileName):

   292

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   293

                if not os.path.exists(mo_targetDir):

   294

                    mkdir(mo_targetDir)

   295

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir,applicationName,applicationName,langCode)

   296

                if verbose: print cmd

   297

                os.system(cmd)

   298

    os.chdir(currentDir)

   299

   300

# -----------------------------------------------------------------------------

   301

# p r i n t U s a g e         -- Displays how to use this script from the command line --

   302

# ^^^^^^^^^^^^^^^^^^^

   303

#

   304

def printUsage(errorMsg=None) :

   305

    """Displays how to use this script from the command line."""

   306

    print """

   307

    ##################################################################################

   308

    #   mki18n :   Make internationalization files.                                  #

   309

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   310

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   311

    #              Supports C,C++,Python source files.                               #

   312

    #                                                                                #

   313

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   314

    #                                                                                #

   315

    #   Options:                                                                     #

   316

    #     -e               : When -m is used, forces English .mo file creation       #

   317

    #     -h               : prints this help                                        #

   318

    #     -m               : make MO from existing PO files                          #

   319

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   320

    #                        file. Creates a dom_xx.po.new for every existing        #

   321

    #                        language specific .po file. ('xx' stands for the ISO639 #

   322

    #                        two-letter language code and 'dom' stands for the       #

   323

    #                        application domain name).  mki18n requires that you     #

   324

    #                        write a 'app.fil' file  which contains the list of all  #

   325

    #                        source code to parse.                                   #

   326

    #     -v               : verbose (prints comments while running)                 #

   327

    #     --domain=appName : specifies the application domain name.  By default      #

   328

    #                        the directory name is used.                             #

   329

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   330

    #                      If not specified, the target is './locale'                #

   331

    #                                                                                #

   332

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   333

    #   specify the path of the target application.  If you leave it out mki18n      #

   334

    #   will use the current directory as the application main directory.            #        

   335

    #                                                                                #

   336

    ##################################################################################"""

   337

    if errorMsg:

   338

        print "\n   ERROR: %s" % errorMsg

   339

   340

# -----------------------------------------------------------------------------

   341

# f i l e B a s e O f ( )         -- Return base name of filename --

   342

# ^^^^^^^^^^^^^^^^^^^^^^^

   343

# 

   344

def fileBaseOf(filename,withPath=0) :

   345

   """fileBaseOf(filename,withPath) ---> string

   346

   347

   Return base name of filename.  The returned string never includes the extension.

   348

   Use os.path.basename() to return the basename with the extension.  The 

   349

   second argument is optional.  If specified and if set to 'true' (non zero) 

   350

   the string returned contains the full path of the file name.  Otherwise the 

   351

   path is excluded.

   352

   353

   [Example]

   354

   >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   355

   >>> fileBaseOf(fn)

   356

   'test'

   357

   >>> fileBaseOf(fn)

   358

   'test'

   359

   >>> fileBaseOf(fn,1)

   360

   'd:/dev/telepath/tvapp/code/test'

   361

   >>> fileBaseOf(fn,0)

   362

   'test'

   363

   >>> fn = 'abcdef'

   364

   >>> fileBaseOf(fn)

   365

   'abcdef'

   366

   >>> fileBaseOf(fn,1)

   367

   'abcdef'

   368

   >>> fn = "abcdef."

   369

   >>> fileBaseOf(fn)

   370

   'abcdef'

   371

   >>> fileBaseOf(fn,1)

   372

   'abcdef'

   373

   """            

   374

   pos = filename.rfind('.')             

   375

   if pos > 0:

   376

      filename = filename[:pos]

   377

   if withPath:

   378

      return filename

   379

   else:

   380

      return os.path.basename(filename)

   381

# -----------------------------------------------------------------------------

   382

# m k d i r ( )         -- Create a directory (and possibly the entire tree) --

   383

# ^^^^^^^^^^^^^

   384

# 

   385

def mkdir(directory) :

   386

   """Create a directory (and possibly the entire tree).

   387

   388

   The os.mkdir() will fail to create a directory if one of the

   389

   directory in the specified path does not exist.  mkdir()

   390

   solves this problem.  It creates every intermediate directory

   391

   required to create the final path. Under Unix, the function 

   392

   only supports forward slash separator, but under Windows and MacOS

   393

   the function supports the forward slash and the OS separator (backslash

   394

   under windows).

   395

   """ 

   396

   397

   # translate the path separators

   398

   directory = unixpath(directory)

   399

   # build a list of all directory elements

   400

   aList = filter(lambda x: len(x)>0, directory.split('/'))

   401

   theLen = len(aList)                     

   402

   # if the first element is a Windows-style disk drive

   403

   # concatenate it with the first directory

   404

   if aList[0].endswith(':'):

   405

      if theLen > 1:

   406

         aList[1] = aList[0] + '/' + aList[1]

   407

         del aList[0]      

   408

         theLen -= 1         

   409

   # if the original directory starts at root,

   410

   # make sure the first element of the list 

   411

   # starts at root too

   412

   if directory[0] == '/':     

   413

      aList[0] = '/' + aList[0]

   414

   # Now iterate through the list, check if the 

   415

   # directory exists and if not create it

   416

   theDir = ''

   417

   for i in range(theLen):

   418

      theDir += aList[i]

   419

      if not os.path.exists(theDir):

   420

         os.mkdir(theDir)

   421

      theDir += '/'   

   422

   423

# -----------------------------------------------------------------------------

   424

# u n i x p a t h ( )         -- Return a path name that contains Unix separator. --

   425

# ^^^^^^^^^^^^^^^^^^^

   426

# 

   427

def unixpath(thePath) :

   428

   r"""Return a path name that contains Unix separator.

   429

   430

   [Example]

   431

   >>> unixpath(r"d:\test")

   432

   'd:/test'

   433

   >>> unixpath("d:/test/file.txt")

   434

   'd:/test/file.txt'

   435

   >>> 

   436

   """

   437

   thePath = os.path.normpath(thePath)

   438

   if os.sep == '/':

   439

      return thePath

   440

   else:

   441

      return thePath.replace(os.sep,'/')

   442

   443

# ----------------------------------------------------------------------------- 

   444

   445

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   446

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   447

# 

   448

if __name__ == "__main__":

   449

    import getopt     # command line parsing

   450

    argc = len(sys.argv)

   451

    if argc == 1:

   452

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   453

        sys.exit(1)

   454

    # If there is some arguments, parse the command line

   455

    validOptions     = "ehmpv"

   456

    validLongOptions = ['domain=', 'moTarget=']             

   457

    option = {}

   458

    option['forceEnglish'] = 0

   459

    option['mo'] = 0

   460

    option['po'] = 0        

   461

    option['verbose'] = 0

   462

    option['domain'] = None

   463

    option['moTarget'] = None

   464

    try:

   465

        optionList,pargs = getopt.getopt(sys.argv[1:],validOptions,validLongOptions)

   466

    except getopt.GetoptError, e:

   467

        printUsage(e[0])

   468

        sys.exit(1)       

   469

    for (opt,val) in optionList:

   470

        if  (opt == '-h'):    

   471

            printUsage()

   472

            sys.exit(0) 

   473

        elif (opt == '-e'):         option['forceEnglish'] = 1

   474

        elif (opt == '-m'):         option['mo'] = 1

   475

        elif (opt == '-p'):         option['po'] = 1

   476

        elif (opt == '-v'):         option['verbose'] = 1

   477

        elif (opt == '--domain'):   option['domain'] = val

   478

        elif (opt == '--moTarget'): option['moTarget'] = val

   479

    if len(pargs) == 0:

   480

        appDirPath = os.getcwd()

   481

        if option['verbose']:

   482

            print "No project directory given. Using current one:  %s" % appDirPath

   483

    elif len(pargs) == 1:

   484

        appDirPath = pargs[0]

   485

    else:

   486

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   487

        sys.exit(1)

   488

    if option['domain'] is None:

   489

        # If no domain specified, use the name of the target directory

   490

        option['domain'] = fileBaseOf(appDirPath)

   491

    if option['verbose']:

   492

        print "Application domain used is: '%s'" % option['domain']

   493

    if option['po']:

   494

        try:

   495

            makePO(appDirPath,option['domain'],option['verbose'])

   496

        except IOError, e:

   497

            printUsage(e[1] + '\n   You must write a file app.fil that contains the list of all files to parse.')

   498

    if option['mo']:

   499

        makeMO(appDirPath,option['moTarget'],option['domain'],option['verbose'],option['forceEnglish'])

   500

    sys.exit(1)            

   501

   502

   503

# -----------------------------------------------------------------------------