1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

# 

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

# 

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

# 

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

# 

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

# 

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

# 

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

# 

    17

#   Update history:

    18

# 

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

# 

    31

#   RCS $Log: $

    32

# 

    33

# 

    34

# -----------------------------------------------------------------------------

    35

"""                                

    36

mki18n allows you to internationalize your software.  You can use it to 

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see 

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files: 

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_ 

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_ 

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip           

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip 

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

# 

    81

import os

    82

import sys

    83

import wx

    84

import re

    85

    86

# -----------------------------------------------------------------------------

    87

# Global variables

    88

# ----------------

    89

#

    90

    91

__author__ = "Pierre Rouleau"

    92

__version__= "$Revision: 1.5 $"

    93

    94

# -----------------------------------------------------------------------------

    95

    96

def getlanguageDict():

    97

    languageDict = {}

    98

    99

    if wx.VERSION >= (3, 0, 0):

   100

        app = wx.App()

   101

    else:

   102

        app = wx.PySimpleApp()

   103

   104

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   105

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   106

        if i:

   107

            languageDict[i.CanonicalName] = i.Description

   108

   109

    return languageDict

   110

   111

   112

   113

def processCustomFiles(filein, fileout, regexp, prefix = ''):

   114

    appfil_file = open(filein, 'r')

   115

    messages_file = open(fileout, 'r')

   116

    messages = messages_file.read()

   117

    messages_file.close()

   118

    messages_file = open(fileout, 'a')

   119

    messages_file.write('\n')

   120

    messages_file.write('#: %s\n' % prefix)

   121

    messages_file.write('\n')

   122

   123

    words_found = {}

   124

    for filepath in appfil_file.xreadlines():

   125

        code_file = open(filepath.strip(), 'r')

   126

        for match in regexp.finditer(code_file.read()):

   127

            word = match.group(1)

   128

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   129

                words_found[word] = True

   130

                messages_file.write('\n')

   131

                messages_file.write("msgid \"%s\"\n"%word)

   132

                messages_file.write("msgstr \"\"\n")

   133

        code_file.close()

   134

   135

    messages_file.close()

   136

    appfil_file.close()

   137

   138

   139

# -----------------------------------------------------------------------------

   140

# m a k e P O ( )         -- Build the Portable Object file for the application --

   141

# ^^^^^^^^^^^^^^^

   142

#

   143

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0) :

   144

    """Build the Portable Object Template file for the application.

   145

   146

    makePO builds the .pot file for the application stored inside 

   147

    a specified directory by running xgettext for all application source 

   148

    files.  It finds the name of all files by looking for a file called 'app.fil'. 

   149

    If this file does not exists, makePo raises an IOError exception.

   150

    By default the application domain (the application

   151

    name) is the same as the directory name but it can be overridden by the

   152

    'applicationDomain' argument.

   153

   154

    makePO always creates a new file called messages.pot.  If it finds files 

   155

    of the form app_xx.po where 'app' is the application name and 'xx' is one 

   156

    of the ISO 639 two-letter language codes, makePO resynchronizes those 

   157

    files with the latest extracted strings (now contained in messages.pot). 

   158

    This process updates all line location number in the language-specific

   159

    .po files and may also create new entries for translation (or comment out 

   160

    some).  The .po file is not changed, instead a new file is created with 

   161

    the .new extension appended to the name of the .po file.

   162

   163

    By default the function does not display what it is doing.  Set the 

   164

    verbose argument to 1 to force it to print its commands.

   165

    """

   166

   167

    if applicationDomain is None:

   168

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   169

    else:

   170

        applicationName = applicationDomain

   171

    currentDir = os.getcwd()

   172

    os.chdir(applicationDirectoryPath)

   173

    filelist = 'app.fil'

   174

    if not os.path.exists(filelist):

   175

        raise IOError(2,'No module file: ' % filelist)

   176

   177

    fileout = 'messages.pot'

   178

    # Steps:                                  

   179

    #  Use xgettext to parse all application modules

   180

    #  The following switches are used:

   181

    #  

   182

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   183

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   184

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   185

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   186

    if verbose: print cmd

   187

    os.system(cmd)

   188

   189

    cmd = 'sed --in-place ' + fileout + ' --expression=s/CHARSET/UTF-8/'

   190

    if verbose: print cmd

   191

    os.system(cmd)

   192

   193

    XSD_STRING_MODEL = re.compile("<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   194

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   195

   196

    XML_TC6_STRING_MODEL = re.compile("<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   197

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')    

   198

   199

    languageDict = getlanguageDict()

   200

   201

    for langCode in languageDict.keys():

   202

        if langCode == 'en':

   203

            pass

   204

        else:

   205

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   206

            if os.path.exists(langPOfileName):

   207

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   208

                if verbose: print cmd

   209

                os.system(cmd)

   210

    os.chdir(currentDir)

   211

   212

# -----------------------------------------------------------------------------

   213

# c a t P O ( )         -- Concatenate one or several PO files with the application domain files. --

   214

# ^^^^^^^^^^^^^

   215

#

   216

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0) :

   217

    """Concatenate one or several PO files with the application domain files.

   218

    """

   219

   220

    if applicationDomain is None:

   221

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   222

    else:

   223

        applicationName = applicationDomain

   224

    currentDir = os.getcwd()

   225

    os.chdir(applicationDirectoryPath)

   226

   227

    languageDict = getlanguageDict()

   228

   229

    for langCode in languageDict.keys():

   230

        if langCode == 'en':

   231

            pass

   232

        else:

   233

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   234

            if os.path.exists(langPOfileName):

   235

                fileList = ''

   236

                for fileName in listOf_extraPo:

   237

                    fileList += ("%s_%s.po " % (fileName,langCode))

   238

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   239

                if verbose: print cmd

   240

                os.system(cmd)

   241

                if targetDir is None:

   242

                    pass

   243

                else:

   244

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   245

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir,applicationName,applicationName,langCode)

   246

                    if verbose: print cmd

   247

                    os.system(cmd)

   248

    os.chdir(currentDir)

   249

   250

# -----------------------------------------------------------------------------

   251

# m a k e M O ( )         -- Compile the Portable Object files into the Machine Object stored in the right location. --

   252

# ^^^^^^^^^^^^^^^

   253

# 

   254

def makeMO(applicationDirectoryPath,targetDir='./locale',applicationDomain=None, verbose=0, forceEnglish=0) :

   255

    """Compile the Portable Object files into the Machine Object stored in the right location.

   256

   257

    makeMO converts all translated language-specific PO files located inside 

   258

    the  application directory into the binary .MO files stored inside the 

   259

    LC_MESSAGES sub-directory for the found locale files.

   260

   261

    makeMO searches for all files that have a name of the form 'app_xx.po' 

   262

    inside the application directory specified by the first argument.  The 

   263

    'app' is the application domain name (that can be specified by the 

   264

    applicationDomain argument or is taken from the directory name). The 'xx' 

   265

    corresponds to one of the ISO 639 two-letter language codes.

   266

   267

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   268

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   269

    code.

   270

    """

   271

    if targetDir is None:

   272

        targetDir = './locale'

   273

    if verbose:

   274

        print "Target directory for .mo files is: %s" % targetDir

   275

   276

    if applicationDomain is None:

   277

        applicationName = fileBaseOf(applicationDirectoryPath,withPath=0)

   278

    else:

   279

        applicationName = applicationDomain

   280

    currentDir = os.getcwd()

   281

    os.chdir(applicationDirectoryPath)

   282

   283

    languageDict = getlanguageDict()

   284

   285

    for langCode in languageDict.keys():

   286

        if (langCode == 'en') and (forceEnglish==0):

   287

            pass

   288

        else:

   289

            langPOfileName = "%s_%s.po" % (applicationName , langCode)

   290

            if os.path.exists(langPOfileName):

   291

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir,langCode)

   292

                if not os.path.exists(mo_targetDir):

   293

                    mkdir(mo_targetDir)

   294

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir,applicationName,applicationName,langCode)

   295

                if verbose: print cmd

   296

                os.system(cmd)

   297

    os.chdir(currentDir)

   298

   299

# -----------------------------------------------------------------------------

   300

# p r i n t U s a g e         -- Displays how to use this script from the command line --

   301

# ^^^^^^^^^^^^^^^^^^^

   302

#

   303

def printUsage(errorMsg=None) :

   304

    """Displays how to use this script from the command line."""

   305

    print """

   306

    ##################################################################################

   307

    #   mki18n :   Make internationalization files.                                  #

   308

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   309

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   310

    #              Supports C,C++,Python source files.                               #

   311

    #                                                                                #

   312

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   313

    #                                                                                #

   314

    #   Options:                                                                     #

   315

    #     -e               : When -m is used, forces English .mo file creation       #

   316

    #     -h               : prints this help                                        #

   317

    #     -m               : make MO from existing PO files                          #

   318

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   319

    #                        file. Creates a dom_xx.po.new for every existing        #

   320

    #                        language specific .po file. ('xx' stands for the ISO639 #

   321

    #                        two-letter language code and 'dom' stands for the       #

   322

    #                        application domain name).  mki18n requires that you     #

   323

    #                        write a 'app.fil' file  which contains the list of all  #

   324

    #                        source code to parse.                                   #

   325

    #     -v               : verbose (prints comments while running)                 #

   326

    #     --domain=appName : specifies the application domain name.  By default      #

   327

    #                        the directory name is used.                             #

   328

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   329

    #                      If not specified, the target is './locale'                #

   330

    #                                                                                #

   331

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   332

    #   specify the path of the target application.  If you leave it out mki18n      #

   333

    #   will use the current directory as the application main directory.            #        

   334

    #                                                                                #

   335

    ##################################################################################"""

   336

    if errorMsg:

   337

        print "\n   ERROR: %s" % errorMsg

   338

   339

# -----------------------------------------------------------------------------

   340

# f i l e B a s e O f ( )         -- Return base name of filename --

   341

# ^^^^^^^^^^^^^^^^^^^^^^^

   342

# 

   343

def fileBaseOf(filename,withPath=0) :

   344

   """fileBaseOf(filename,withPath) ---> string

   345

   346

   Return base name of filename.  The returned string never includes the extension.

   347

   Use os.path.basename() to return the basename with the extension.  The 

   348

   second argument is optional.  If specified and if set to 'true' (non zero) 

   349

   the string returned contains the full path of the file name.  Otherwise the 

   350

   path is excluded.

   351

   352

   [Example]

   353

   >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   354

   >>> fileBaseOf(fn)

   355

   'test'

   356

   >>> fileBaseOf(fn)

   357

   'test'

   358

   >>> fileBaseOf(fn,1)

   359

   'd:/dev/telepath/tvapp/code/test'

   360

   >>> fileBaseOf(fn,0)

   361

   'test'

   362

   >>> fn = 'abcdef'

   363

   >>> fileBaseOf(fn)

   364

   'abcdef'

   365

   >>> fileBaseOf(fn,1)

   366

   'abcdef'

   367

   >>> fn = "abcdef."

   368

   >>> fileBaseOf(fn)

   369

   'abcdef'

   370

   >>> fileBaseOf(fn,1)

   371

   'abcdef'

   372

   """            

   373

   pos = filename.rfind('.')             

   374

   if pos > 0:

   375

      filename = filename[:pos]

   376

   if withPath:

   377

      return filename

   378

   else:

   379

      return os.path.basename(filename)

   380

# -----------------------------------------------------------------------------

   381

# m k d i r ( )         -- Create a directory (and possibly the entire tree) --

   382

# ^^^^^^^^^^^^^

   383

# 

   384

def mkdir(directory) :

   385

   """Create a directory (and possibly the entire tree).

   386

   387

   The os.mkdir() will fail to create a directory if one of the

   388

   directory in the specified path does not exist.  mkdir()

   389

   solves this problem.  It creates every intermediate directory

   390

   required to create the final path. Under Unix, the function 

   391

   only supports forward slash separator, but under Windows and MacOS

   392

   the function supports the forward slash and the OS separator (backslash

   393

   under windows).

   394

   """ 

   395

   396

   # translate the path separators

   397

   directory = unixpath(directory)

   398

   # build a list of all directory elements

   399

   aList = filter(lambda x: len(x)>0, directory.split('/'))

   400

   theLen = len(aList)                     

   401

   # if the first element is a Windows-style disk drive

   402

   # concatenate it with the first directory

   403

   if aList[0].endswith(':'):

   404

      if theLen > 1:

   405

         aList[1] = aList[0] + '/' + aList[1]

   406

         del aList[0]      

   407

         theLen -= 1         

   408

   # if the original directory starts at root,

   409

   # make sure the first element of the list 

   410

   # starts at root too

   411

   if directory[0] == '/':     

   412

      aList[0] = '/' + aList[0]

   413

   # Now iterate through the list, check if the 

   414

   # directory exists and if not create it

   415

   theDir = ''

   416

   for i in range(theLen):

   417

      theDir += aList[i]

   418

      if not os.path.exists(theDir):

   419

         os.mkdir(theDir)

   420

      theDir += '/'   

   421

   422

# -----------------------------------------------------------------------------

   423

# u n i x p a t h ( )         -- Return a path name that contains Unix separator. --

   424

# ^^^^^^^^^^^^^^^^^^^

   425

# 

   426

def unixpath(thePath) :

   427

   r"""Return a path name that contains Unix separator.

   428

   429

   [Example]

   430

   >>> unixpath(r"d:\test")

   431

   'd:/test'

   432

   >>> unixpath("d:/test/file.txt")

   433

   'd:/test/file.txt'

   434

   >>> 

   435

   """

   436

   thePath = os.path.normpath(thePath)

   437

   if os.sep == '/':

   438

      return thePath

   439

   else:

   440

      return thePath.replace(os.sep,'/')

   441

   442

# ----------------------------------------------------------------------------- 

   443

   444

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   445

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   446

# 

   447

if __name__ == "__main__":

   448

    import getopt     # command line parsing

   449

    argc = len(sys.argv)

   450

    if argc == 1:

   451

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   452

        sys.exit(1)

   453

    # If there is some arguments, parse the command line

   454

    validOptions     = "ehmpv"

   455

    validLongOptions = ['domain=', 'moTarget=']             

   456

    option = {}

   457

    option['forceEnglish'] = 0

   458

    option['mo'] = 0

   459

    option['po'] = 0        

   460

    option['verbose'] = 0

   461

    option['domain'] = None

   462

    option['moTarget'] = None

   463

    try:

   464

        optionList,pargs = getopt.getopt(sys.argv[1:],validOptions,validLongOptions)

   465

    except getopt.GetoptError, e:

   466

        printUsage(e[0])

   467

        sys.exit(1)       

   468

    for (opt,val) in optionList:

   469

        if  (opt == '-h'):    

   470

            printUsage()

   471

            sys.exit(0) 

   472

        elif (opt == '-e'):         option['forceEnglish'] = 1

   473

        elif (opt == '-m'):         option['mo'] = 1

   474

        elif (opt == '-p'):         option['po'] = 1

   475

        elif (opt == '-v'):         option['verbose'] = 1

   476

        elif (opt == '--domain'):   option['domain'] = val

   477

        elif (opt == '--moTarget'): option['moTarget'] = val

   478

    if len(pargs) == 0:

   479

        appDirPath = os.getcwd()

   480

        if option['verbose']:

   481

            print "No project directory given. Using current one:  %s" % appDirPath

   482

    elif len(pargs) == 1:

   483

        appDirPath = pargs[0]

   484

    else:

   485

        printUsage('Too many arguments (%u).  Use double quotes if you have space in directory name' % len(pargs))

   486

        sys.exit(1)

   487

    if option['domain'] is None:

   488

        # If no domain specified, use the name of the target directory

   489

        option['domain'] = fileBaseOf(appDirPath)

   490

    if option['verbose']:

   491

        print "Application domain used is: '%s'" % option['domain']

   492

    if option['po']:

   493

        try:

   494

            makePO(appDirPath,option['domain'],option['verbose'])

   495

        except IOError, e:

   496

            printUsage(e[1] + '\n   You must write a file app.fil that contains the list of all files to parse.')

   497

    if option['mo']:

   498

        makeMO(appDirPath,option['moTarget'],option['domain'],option['verbose'],option['forceEnglish'])

   499

    sys.exit(1)            

   500

   501

   502

# -----------------------------------------------------------------------------