1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

# 

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

# 

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

# 

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

# 

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

# 

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

# 

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

# 

    17

#   Update history:

    18

# 

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

# 

    31

#   RCS $Log: $

    32

# 

    33

# 

    34

# -----------------------------------------------------------------------------

    35

"""                                

    36

mki18n allows you to internationalize your software.  You can use it to 

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see 

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files: 

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_ 

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_ 

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip           

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip 

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

# 

    81

import os

    82

import sys

    83

import wx

    84

import re

    85

    86

# -----------------------------------------------------------------------------

    87

# Global variables

    88

# ----------------

    89

#

    90

    91

__author__ = "Pierre Rouleau"

    92

__version__= "$Revision: 1.5 $"

    93

    94

# -----------------------------------------------------------------------------

    95

    96

def getlanguageDict():

    97

    languageDict = {}

    98

    99

    if wx.VERSION >= (3, 0, 0):

   100

        app = wx.App()

   101

    else:

   102

        app = wx.PySimpleApp()

   103

   104

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   105

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   106

        if i:

   107

            languageDict[i.CanonicalName] = i.Description

   108

   109

    return languageDict

   110

   111

   112

   113

def processCustomFiles(filein, fileout, regexp, prefix = ''):

   114

    appfil_file = open(filein, 'r')

   115

    messages_file = open(fileout, 'r')

   116

    messages = messages_file.read()

   117

    messages_file.close()

   118

    messages_file = open(fileout, 'a')

   119

    messages_file.write('\n')

   120

    messages_file.write('#: %s\n' % prefix)

   121

    messages_file.write('\n')

   122

   123

    words_found = {}

   124

    for filepath in appfil_file.xreadlines():

   125

        code_file = open(filepath.strip(), 'r')

   126

        for match in regexp.finditer(code_file.read()):

   127

            word = match.group(1)

   128

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   129

                words_found[word] = True

   130

                messages_file.write('\n')

   131

                messages_file.write("msgid \"%s\"\n"%word)

   132

                messages_file.write("msgstr \"\"\n")

   133

        code_file.close()

   134

   135

    messages_file.close()

   136

    appfil_file.close()

   137

   138

   139

# -----------------------------------------------------------------------------

   140

# m a k e P O ( )         -- Build the Portable Object file for the application --

   141

# ^^^^^^^^^^^^^^^

   142

#

   143

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0) :

   144

    """Build the Portable Object Template file for the application.

   145

   146

    makePO builds the .pot file for the application stored inside 

   147

    a specified directory by running xgettext for all application source 

   148

    files.  It finds the name of all files by looking for a file called 'app.fil'. 

   149

    If this file does not exists, makePo raises an IOError exception.

   150

    By default the application domain (the application

   151

    name) is the same as the directory name but it can be overridden by the

   152

    'applicationDomain' argument.

   153

   154

    makePO always creates a new file called messages.pot.  If it finds files 

   155

    of the form app_xx.po where 'app' is the application name and 'xx' is one 

   156

    of the ISO 639 two-letter language codes, makePO resynchronizes those 

   157

    files with the latest extracted strings (now contained in messages.pot). 

   158

    This process updates all line location number in the language-specific

   159

    .po files and may also create new entries for translation (or comment out 

   160

    some).  The .po file is not changed, instead a new file is created with 

   161

    the .new extension appended to the name of the .po file.

   162

   163

    By default the function does not display what it is doing.  Set the 

   164

    verbose argument to 1 to force it to print its commands.

   165

    """

   166

   167

    if applicationDomain is None:

   168

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   169

    else:

   170

        applicationName = applicationDomain

   171

    currentDir = os.getcwd()

   172

    os.chdir(applicationDirectoryPath)

   173

    filelist = 'app.fil'

   174

    if not os.path.exists(filelist):

   175

        raise IOError(2,'No module file: ' % filelist)

   176

   177

    fileout = 'messages.pot'

   178

    # Steps:                                  

   179

    #  Use xgettext to parse all application modules

   180

    #  The following switches are used:

   181

    #  

   182

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   183

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   184

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   185

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout

   186

    if verbose: print cmd

   187

    os.system(cmd)                                                

   188

   189

    XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   190

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   191

   192

    XML_TC6_STRING_MODEL = re.compile("<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   193

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')    

   194

   195

    languageDict = getlanguageDict()

   196

   197

    for langCode in languageDict.keys():

   198

        if langCode == 'en':

   199

            pass

   200

        else:

   201

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   202

            if os.path.exists(langPOfileName):

   203

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   204

                if verbose: print cmd

   205

                os.system(cmd)

   206

    os.chdir(currentDir)

   207

   208

# -----------------------------------------------------------------------------

   209

# c a t P O ( )         -- Concatenate one or several PO files with the application domain files. --

   210

# ^^^^^^^^^^^^^

   211

#

   212

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0) :

   213

    """Concatenate one or several PO files with the application domain files.

   214

    """

   215

   216

    if applicationDomain is None:

   217

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   218

    else:

   219

        applicationName = applicationDomain

   220

    currentDir = os.getcwd()

   221

    os.chdir(applicationDirectoryPath)

   222

   223

    languageDict = getlanguageDict()

   224

   225

    for langCode in languageDict.keys():

   226

        if langCode == 'en':

   227

            pass

   228

        else:

   229

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   230

            if os.path.exists(langPOfileName):

   231

                fileList = ''

   232

                for fileName in listOf_extraPo:

   233

                    fileList += ("%s_%s.po " % (fileName,langCode))

   234

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   235

                if verbose: print cmd

   236

                os.system(cmd)

   237

                if targetDir is None:

   238

                    pass

   239

                else:

   240

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   241

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir,applicationName,applicationName,langCode)

   242

                    if verbose: print cmd

   243

                    os.system(cmd)

   244

    os.chdir(currentDir)

   245

   246

# -----------------------------------------------------------------------------

   247

# m a k e M O ( )         -- Compile the Portable Object files into the Machine Object stored in the right location. --

   248

# ^^^^^^^^^^^^^^^

   249

# 

   250

def makeMO(applicationDirectoryPath,targetDir='./locale',applicationDomain=None, verbose=0, forceEnglish=0) :

   251

    """Compile the Portable Object files into the Machine Object stored in the right location.

   252

   253

    makeMO converts all translated language-specific PO files located inside 

   254

    the  application directory into the binary .MO files stored inside the 

   255

    LC_MESSAGES sub-directory for the found locale files.

   256

   257

    makeMO searches for all files that have a name of the form 'app_xx.po' 

   258

    inside the application directory specified by the first argument.  The 

   259

    'app' is the application domain name (that can be specified by the 

   260

    applicationDomain argument or is taken from the directory name). The 'xx' 

   261

    corresponds to one of the ISO 639 two-letter language codes.

   262

   263

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   264

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   265

    code.

   266

    """

   267

    if targetDir is None:

   268

        targetDir = './locale'

   269

    if verbose:

   270

        print "Target directory for .mo files is: %s" % targetDir

   271

   272

    if applicationDomain is None:

   273

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   274

    else:

   275

        applicationName = applicationDomain

   276

    currentDir = os.getcwd()

   277

    os.chdir(applicationDirectoryPath)

   278

   279

    languageDict = getlanguageDict()

   280

   281

    for langCode in languageDict.keys():

   282

        if (langCode == 'en') and (forceEnglish==0):

   283

            pass

   284

        else:

   285

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   286

            if os.path.exists(langPOfileName):

   287

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   288

                if not os.path.exists(mo_targetDir):

   289

                    mkdir(mo_targetDir)

   290

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir,applicationName,applicationName,langCode)

   291

                if verbose: print cmd

   292

                os.system(cmd)

   293

    os.chdir(currentDir)

   294

   295

# -----------------------------------------------------------------------------

   296

# p r i n t U s a g e         -- Displays how to use this script from the command line --

   297

# ^^^^^^^^^^^^^^^^^^^

   298

#

   299

def printUsage(errorMsg=None) :

   300

    """Displays how to use this script from the command line."""

   301

    print """

   302

    ##################################################################################

   303

    #   mki18n :   Make internationalization files.                                  #

   304

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   305

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   306

    #              Supports C,C++,Python source files.                               #

   307

    #                                                                                #

   308

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   309

    #                                                                                #

   310

    #   Options:                                                                     #

   311

    #     -e               : When -m is used, forces English .mo file creation       #

   312

    #     -h               : prints this help                                        #

   313

    #     -m               : make MO from existing PO files                          #

   314

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   315

    #                        file. Creates a dom_xx.po.new for every existing        #

   316

    #                        language specific .po file. ('xx' stands for the ISO639 #

   317

    #                        two-letter language code and 'dom' stands for the       #

   318

    #                        application domain name).  mki18n requires that you     #

   319

    #                        write a 'app.fil' file  which contains the list of all  #

   320

    #                        source code to parse.                                   #

   321

    #     -v               : verbose (prints comments while running)                 #

   322

    #     --domain=appName : specifies the application domain name.  By default      #

   323

    #                        the directory name is used.                             #

   324

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   325

    #                      If not specified, the target is './locale'                #

   326

    #                                                                                #

   327

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   328

    #   specify the path of the target application.  If you leave it out mki18n      #

   329

    #   will use the current directory as the application main directory.            #        

   330

    #                                                                                #

   331

    ##################################################################################"""

   332

    if errorMsg:

   333

        print "\n   ERROR: %s" % errorMsg

   334

   335

# -----------------------------------------------------------------------------

   336

# f i l e B a s e O f ( )         -- Return base name of filename --

   337

# ^^^^^^^^^^^^^^^^^^^^^^^

   338

# 

   339

def fileBaseOf(filename,withPath=0) :

   340

   """fileBaseOf(filename,withPath) ---> string

   341

   342

   Return base name of filename.  The returned string never includes the extension.

   343

   Use os.path.basename() to return the basename with the extension.  The 

   344

   second argument is optional.  If specified and if set to 'true' (non zero) 

   345

   the string returned contains the full path of the file name.  Otherwise the 

   346

   path is excluded.

   347

   348

   [Example]

   349

   >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   350

   >>> fileBaseOf(fn)

   351

   'test'

   352

   >>> fileBaseOf(fn)

   353

   'test'

   354

   >>> fileBaseOf(fn,1)

   355

   'd:/dev/telepath/tvapp/code/test'

   356

   >>> fileBaseOf(fn,0)

   357

   'test'

   358

   >>> fn = 'abcdef'

   359

   >>> fileBaseOf(fn)

   360

   'abcdef'

   361

   >>> fileBaseOf(fn,1)

   362

   'abcdef'

   363

   >>> fn = "abcdef."

   364

   >>> fileBaseOf(fn)

   365

   'abcdef'

   366

   >>> fileBaseOf(fn,1)

   367

   'abcdef'

   368

   """            

   369

   pos = filename.rfind('.')             

   370

   if pos > 0:

   371

      filename = filename[:pos]

   372

   if withPath:

   373

      return filename

   374

   else:

   375

      return os.path.basename(filename)

   376

# -----------------------------------------------------------------------------

   377

# m k d i r ( )         -- Create a directory (and possibly the entire tree) --

   378

# ^^^^^^^^^^^^^

   379

# 

   380

def mkdir(directory) :

   381

   """Create a directory (and possibly the entire tree).

   382

   383

   The os.mkdir() will fail to create a directory if one of the

   384

   directory in the specified path does not exist.  mkdir()

   385

   solves this problem.  It creates every intermediate directory

   386

   required to create the final path. Under Unix, the function 

   387

   only supports forward slash separator, but under Windows and MacOS

   388

   the function supports the forward slash and the OS separator (backslash

   389

   under windows).

   390

   """ 

   391

   392

   # translate the path separators

   393

   directory = unixpath(directory)

   394

   # build a list of all directory elements

   395

   aList = filter(lambda x: len(x)>0, directory.split('/'))

   396

   theLen = len(aList)                     

   397

   # if the first element is a Windows-style disk drive

   398

   # concatenate it with the first directory

   399

   if aList[0].endswith(':'):

   400

      if theLen > 1:

   401

         aList[1] = aList[0] + '/' + aList[1]

   402

         del aList[0]      

   403

         theLen -= 1         

   404

   # if the original directory starts at root,

   405

   # make sure the first element of the list 

   406

   # starts at root too

   407

   if directory[0] == '/':     

   408

      aList[0] = '/' + aList[0]

   409

   # Now iterate through the list, check if the 

   410

   # directory exists and if not create it

   411

   theDir = ''

   412

   for i in range(theLen):

   413

      theDir += aList[i]

   414

      if not os.path.exists(theDir):

   415

         os.mkdir(theDir)

   416

      theDir += '/'   

   417

   418

# -----------------------------------------------------------------------------

   419

# u n i x p a t h ( )         -- Return a path name that contains Unix separator. --

   420

# ^^^^^^^^^^^^^^^^^^^

   421

# 

   422

def unixpath(thePath) :

   423

   r"""Return a path name that contains Unix separator.

   424

   425

   [Example]

   426

   >>> unixpath(r"d:\test")

   427

   'd:/test'

   428

   >>> unixpath("d:/test/file.txt")

   429

   'd:/test/file.txt'

   430

   >>> 

   431

   """

   432

   thePath = os.path.normpath(thePath)

   433

   if os.sep == '/':

   434

      return thePath

   435

   else:

   436

      return thePath.replace(os.sep,'/')

   437

   438

# ----------------------------------------------------------------------------- 

   439

   440

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   441

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   442

# 

   443

if __name__ == "__main__":

   444

    import getopt     # command line parsing

   445

    argc = len(sys.argv)

   446

    if argc == 1:

   447

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   448

        sys.exit(1)

   449

    # If there is some arguments, parse the command line

   450

    validOptions     = "ehmpv"

   451

    validLongOptions = ['domain=', 'moTarget=']             

   452

    option = {}

   453

    option['forceEnglish'] = 0

   454

    option['mo'] = 0

   455

    option['po'] = 0        

   456

    option['verbose'] = 0

   457

    option['domain'] = None

   458

    option['moTarget'] = None

   459

    try:

   460

        optionList,pargs = getopt.getopt(sys.argv[1:],validOptions,validLongOptions)

   461

    except getopt.GetoptError, e:

   462

        printUsage(e[0])

   463

        sys.exit(1)       

   464

    for (opt,val) in optionList:

   465

        if  (opt == '-h'):    

   466

            printUsage()

   467

            sys.exit(0) 

   468

        elif (opt == '-e'):         option['forceEnglish'] = 1

   469

        elif (opt == '-m'):         option['mo'] = 1

   470

        elif (opt == '-p'):         option['po'] = 1

   471

        elif (opt == '-v'):         option['verbose'] = 1

   472

        elif (opt == '--domain'):   option['domain'] = val

   473

        elif (opt == '--moTarget'): option['moTarget'] = val

   474

    if len(pargs) == 0:

   475

        appDirPath = os.getcwd()

   476

        if option['verbose']:

   477

            print "No project directory given. Using current one:  %s" % appDirPath

   478

    elif len(pargs) == 1:

   479

        appDirPath = pargs[0]

   480

    else:

   481

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   482

        sys.exit(1)

   483

    if option['domain'] is None:

   484

        # If no domain specified, use the name of the target directory

   485

        option['domain'] = fileBaseOf(appDirPath)

   486

    if option['verbose']:

   487

        print "Application domain used is: '%s'" % option['domain']

   488

    if option['po']:

   489

        try:

   490

            makePO(appDirPath,option['domain'],option['verbose'])

   491

        except IOError, e:

   492

            printUsage(e[1] + '\n   You must write a file app.fil that contains the list of all files to parse.')

   493

    if option['mo']:

   494

        makeMO(appDirPath,option['moTarget'],option['domain'],option['verbose'],option['forceEnglish'])

   495

    sys.exit(1)            

   496

   497

   498

# -----------------------------------------------------------------------------