1

#! /usr/bin/env python

     2

# -*- coding: iso-8859-1 -*-

     3

#

     4

#   PYTHON MODULE:     MKI18N.PY

     5

#                      =========

     6

#

     7

#   Abstract:         Make Internationalization (i18n) files for an application.

     8

#

     9

#   Copyright Pierre Rouleau. 2003. Released to public domain.

    10

#

    11

#   Last update: Saturday, November 8, 2003. @ 15:55:18.

    12

#

    13

#   File: ROUP2003N01::C:/dev/python/mki18n.py

    14

#

    15

#   RCS $Header: //software/official/MKS/MKS_SI/TV_NT/dev/Python/rcs/mki18n.py 1.5 2003/11/05 19:40:04 PRouleau Exp $

    16

#

    17

#   Update history:

    18

#

    19

#   - File created: Saturday, June 7, 2003. by Pierre Rouleau

    20

#   - 10/06/03 rcs : RCS Revision 1.1  2003/06/10 10:06:12  PRouleau

    21

#   - 10/06/03 rcs : RCS Initial revision

    22

#   - 23/08/03 rcs : RCS Revision 1.2  2003/06/10 10:54:27  PRouleau

    23

#   - 23/08/03 P.R.: [code:fix] : The strings encoded in this file are encode in iso-8859-1 format.  Added the encoding

    24

#                    notification to Python to comply with Python's 2.3 PEP 263.

    25

#   - 23/08/03 P.R.: [feature:new] : Added the '-e' switch which is used to force the creation of the empty English .mo file.

    26

#   - 22/10/03 P.R.: [code] : incorporated utility functions in here to make script self sufficient.

    27

#   - 05/11/03 rcs : RCS Revision 1.4  2003/10/22 06:39:31  PRouleau

    28

#   - 05/11/03 P.R.: [code:fix] : included the unixpath() in this file.

    29

#   - 08/11/03 rcs : RCS Revision 1.5  2003/11/05 19:40:04  PRouleau

    30

#

    31

#   RCS $Log: $

    32

#

    33

#

    34

# -----------------------------------------------------------------------------

    35

"""

    36

mki18n allows you to internationalize your software.  You can use it to

    37

create the GNU .po files (Portable Object) and the compiled .mo files

    38

(Machine Object).

    39

    40

mki18n module can be used from the command line or from within a script (see

    41

the Usage at the end of this page).

    42

    43

    Table of Contents

    44

    -----------------

    45

    46

    makePO()             -- Build the Portable Object file for the application --

    47

    catPO()              -- Concatenate one or several PO files with the application domain files. --

    48

    makeMO()             -- Compile the Portable Object files into the Machine Object stored in the right location. --

    49

    printUsage           -- Displays how to use this script from the command line --

    50

    51

    Scriptexecution      -- Runs when invoked from the command line --

    52

    53

    54

NOTE:  this module uses GNU gettext utilities.

    55

    56

You can get the gettext tools from the following sites:

    57

    58

   - `GNU FTP site for gettetx`_ where several versions (0.10.40, 0.11.2, 0.11.5 and 0.12.1) are available.

    59

     Note  that you need to use `GNU libiconv`_ to use this. Get it from the `GNU

    60

     libiconv  ftp site`_ and get version 1.9.1 or later. Get the Windows .ZIP

    61

     files and install the packages inside c:/gnu. All binaries will be stored

    62

     inside  c:/gnu/bin.  Just  put c:/gnu/bin inside your PATH. You will need

    63

     the following files:

    64

    65

      - `gettext-runtime-0.12.1.bin.woe32.zip`_

    66

      - `gettext-tools-0.12.1.bin.woe32.zip`_

    67

      - `libiconv-1.9.1.bin.woe32.zip`_

    68

    69

    70

.. _GNU libiconv:                            http://www.gnu.org/software/libiconv/

    71

.. _GNU libiconv ftp site:                   http://www.ibiblio.org/pub/gnu/libiconv/

    72

.. _gettext-runtime-0.12.1.bin.woe32.zip:    ftp://ftp.gnu.org/gnu/gettext/gettext-runtime-0.12.1.bin.woe32.zip

    73

.. _gettext-tools-0.12.1.bin.woe32.zip:      ftp://ftp.gnu.org/gnu/gettext/gettext-tools-0.12.1.bin.woe32.zip

    74

.. _libiconv-1.9.1.bin.woe32.zip:            http://www.ibiblio.org/pub/gnu/libiconv/libiconv-1.9.1.bin.woe32.zip

    75

    76

"""

    77

# -----------------------------------------------------------------------------

    78

# Module Import

    79

# -------------

    80

#

    81

    82

    83

import os

    84

import sys

    85

import re

    86

import wx

    87

    88

    89

# -----------------------------------------------------------------------------

    90

# Global variables

    91

# ----------------

    92

#

    93

    94

__author__ = "Pierre Rouleau"

    95

__version__ = "$Revision: 1.5 $"

    96

    97

# -----------------------------------------------------------------------------

    98

    99

   100

def getSupportedLanguageDict(appname):

   101

    """

   102

    Returns dictionary with languages already supported

   103

    by given application

   104

   105

    param: appname:

   106

        name of application

   107

    """

   108

    languageDict = {}

   109

    ext = '.po'

   110

    files = [x for x in os.listdir('.')

   111

             if x.startswith(appname) and x.endswith(ext)]

   112

   113

    langs = [x.split(appname + '_')[1].split(ext)[0] for x in files]

   114

    for lang in langs:

   115

        languageDict[lang] = lang

   116

   117

    return languageDict

   118

   119

   120

def getlanguageDict():

   121

    languageDict = {}

   122

    getSupportedLanguageDict('Beremiz')

   123

    _app = wx.App()

   124

   125

    for lang in [x for x in dir(wx) if x.startswith("LANGUAGE")]:

   126

        i = wx.Locale(wx.LANGUAGE_DEFAULT).GetLanguageInfo(getattr(wx, lang))

   127

        if i:

   128

            languageDict[i.CanonicalName] = i.Description

   129

   130

    return languageDict

   131

   132

   133

def verbosePrint(verbose, str):

   134

    if verbose:

   135

        print(str)

   136

   137

   138

def processCustomFiles(filein, fileout, regexp, prefix=''):

   139

    appfil_file = open(filein, 'r')

   140

    messages_file = open(fileout, 'r')

   141

    messages = messages_file.read()

   142

    messages_file.close()

   143

    messages_file = open(fileout, 'a')

   144

    messages_file.write('\n')

   145

    messages_file.write('#: %s\n' % prefix)

   146

    messages_file.write('\n')

   147

   148

    words_found = {}

   149

    for filepath in appfil_file.readlines():

   150

        code_file = open(filepath.strip(), 'r')

   151

        for match in regexp.finditer(code_file.read()):

   152

            word = match.group(1)

   153

            if not words_found.get(word, False) and messages.find("msgid \"%s\"\nmsgstr \"\"" % word) == -1:

   154

                words_found[word] = True

   155

                messages_file.write('\n')

   156

                messages_file.write("msgid \"%s\"\n" % word)

   157

                messages_file.write("msgstr \"\"\n")

   158

        code_file.close()

   159

   160

    messages_file.close()

   161

    appfil_file.close()

   162

   163

   164

# -----------------------------------------------------------------------------

   165

# m a k e P O ( )         -- Build the Portable Object file for the application --

   166

# ^^^^^^^^^^^^^^^

   167

#

   168

def makePO(applicationDirectoryPath,  applicationDomain=None, verbose=0):

   169

    """Build the Portable Object Template file for the application.

   170

   171

    makePO builds the .pot file for the application stored inside

   172

    a specified directory by running xgettext for all application source

   173

    files.  It finds the name of all files by looking for a file called 'app.fil'.

   174

    If this file does not exists, makePo raises an IOError exception.

   175

    By default the application domain (the application

   176

    name) is the same as the directory name but it can be overridden by the

   177

    'applicationDomain' argument.

   178

   179

    makePO always creates a new file called messages.pot.  If it finds files

   180

    of the form app_xx.po where 'app' is the application name and 'xx' is one

   181

    of the ISO 639 two-letter language codes, makePO resynchronizes those

   182

    files with the latest extracted strings (now contained in messages.pot).

   183

    This process updates all line location number in the language-specific

   184

    .po files and may also create new entries for translation (or comment out

   185

    some).  The .po file is not changed, instead a new file is created with

   186

    the .new extension appended to the name of the .po file.

   187

   188

    By default the function does not display what it is doing.  Set the

   189

    verbose argument to 1 to force it to print its commands.

   190

    """

   191

   192

    if applicationDomain is None:

   193

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   194

    else:

   195

        applicationName = applicationDomain

   196

    currentDir = os.getcwd()

   197

    os.chdir(applicationDirectoryPath)

   198

    filelist = 'app.fil'

   199

    if not os.path.exists(filelist):

   200

        raise IOError(2, 'No module file: %s' % filelist)

   201

   202

    fileout = 'messages.pot'

   203

    # Steps:

   204

    #  Use xgettext to parse all application modules

   205

    #  The following switches are used:

   206

    #

   207

    #   -s                          : sort output by string content (easier to use when we need to merge several .po files)

   208

    #   --files-from=app.fil        : The list of files is taken from the file: app.fil

   209

    #   --output=                   : specifies the name of the output file (using a .pot extension)

   210

    cmd = 'xgettext -s --no-wrap --language=Python --files-from=' + filelist + ' --output=' + fileout + ' --package-name ' + applicationName

   211

    verbosePrint(verbose, cmd)

   212

    os.system(cmd)

   213

   214

    XSD_STRING_MODEL = re.compile(r"<xsd\:(?:element|attribute) name=\"([^\"]*)\"[^\>]*\>")

   215

    processCustomFiles(filelist, fileout, XSD_STRING_MODEL, 'Extra XSD strings')

   216

   217

    XML_TC6_STRING_MODEL = re.compile(r"<documentation>\s*<xhtml\:p><!\[CDATA\[([^\]]*)\]\]></xhtml\:p>\s*</documentation>", re.MULTILINE | re.DOTALL)

   218

    processCustomFiles(filelist, fileout, XML_TC6_STRING_MODEL, 'Extra TC6 documentation strings')

   219

   220

    # generate messages.po

   221

    cmd = 'msginit --no-wrap --no-translator -i %s -l en_US.UTF-8 -o messages.po' % (fileout)

   222

    verbosePrint(verbose, cmd)

   223

    os.system(cmd)

   224

   225

    languageDict = getSupportedLanguageDict(applicationName)

   226

   227

    for langCode in list(languageDict.keys()):

   228

        if langCode == 'en':

   229

            pass

   230

        else:

   231

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   232

            if os.path.exists(langPOfileName):

   233

                cmd = 'msgmerge -s --no-wrap "%s" %s > "%s.new"' % (langPOfileName, fileout, langPOfileName)

   234

                verbosePrint(verbose, cmd)

   235

                os.system(cmd)

   236

    os.chdir(currentDir)

   237

   238

   239

def catPO(applicationDirectoryPath, listOf_extraPo, applicationDomain=None, targetDir=None, verbose=0):

   240

    """Concatenate one or several PO files with the application domain files.

   241

    """

   242

   243

    if applicationDomain is None:

   244

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   245

    else:

   246

        applicationName = applicationDomain

   247

    currentDir = os.getcwd()

   248

    os.chdir(applicationDirectoryPath)

   249

   250

    languageDict = getSupportedLanguageDict(applicationName)

   251

   252

    for langCode in list(languageDict.keys()):

   253

        if langCode == 'en':

   254

            pass

   255

        else:

   256

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   257

            if os.path.exists(langPOfileName):

   258

                fileList = ''

   259

                for fileName in listOf_extraPo:

   260

                    fileList += ("%s_%s.po " % (fileName, langCode))

   261

                cmd = "msgcat -s --no-wrap %s %s > %s.cat" % (langPOfileName, fileList, langPOfileName)

   262

                verbosePrint(verbose, cmd)

   263

                os.system(cmd)

   264

                if targetDir is None:

   265

                    pass

   266

                else:

   267

                    mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   268

                    cmd = "msgfmt --output-file=%s/%s.mo %s_%s.po.cat" % (mo_targetDir, applicationName, applicationName, langCode)

   269

                    verbosePrint(verbose, cmd)

   270

                    os.system(cmd)

   271

    os.chdir(currentDir)

   272

   273

   274

def makeMO(applicationDirectoryPath, targetDir='./locale', applicationDomain=None, verbose=0, forceEnglish=0):

   275

    """Compile the Portable Object files into the Machine Object stored in the right location.

   276

   277

    makeMO converts all translated language-specific PO files located inside

   278

    the  application directory into the binary .MO files stored inside the

   279

    LC_MESSAGES sub-directory for the found locale files.

   280

   281

    makeMO searches for all files that have a name of the form 'app_xx.po'

   282

    inside the application directory specified by the first argument.  The

   283

    'app' is the application domain name (that can be specified by the

   284

    applicationDomain argument or is taken from the directory name). The 'xx'

   285

    corresponds to one of the ISO 639 two-letter language codes.

   286

   287

    makeMo stores the resulting files inside a sub-directory of `targetDir`

   288

    called xx/LC_MESSAGES where 'xx' corresponds to the 2-letter language

   289

    code.

   290

    """

   291

   292

    if targetDir is None:

   293

        targetDir = './locale'

   294

   295

    verbosePrint(verbose, "Target directory for .mo files is: %s" % targetDir)

   296

   297

    if applicationDomain is None:

   298

        applicationName = fileBaseOf(applicationDirectoryPath, withPath=0)

   299

    else:

   300

        applicationName = applicationDomain

   301

    currentDir = os.getcwd()

   302

    os.chdir(applicationDirectoryPath)

   303

   304

    languageDict = getSupportedLanguageDict(applicationName)

   305

   306

    for langCode in list(languageDict.keys()):

   307

        if (langCode == 'en') and (forceEnglish == 0):

   308

            pass

   309

        else:

   310

            langPOfileName = "%s_%s.po" % (applicationName, langCode)

   311

            if os.path.exists(langPOfileName):

   312

                mo_targetDir = "%s/%s/LC_MESSAGES" % (targetDir, langCode)

   313

                if not os.path.exists(mo_targetDir):

   314

                    mkdir(mo_targetDir)

   315

                cmd = 'msgfmt --output-file="%s/%s.mo" "%s_%s.po"' % (mo_targetDir, applicationName, applicationName, langCode)

   316

                verbosePrint(verbose, cmd)

   317

                os.system(cmd)

   318

    os.chdir(currentDir)

   319

   320

   321

def printUsage(errorMsg=None):

   322

    """Displays how to use this script from the command line."""

   323

    print("""

   324

    ##################################################################################

   325

    #   mki18n :   Make internationalization files.                                  #

   326

    #              Uses the GNU gettext system to create PO (Portable Object) files  #

   327

    #              from source code, coimpile PO into MO (Machine Object) files.     #

   328

    #              Supports C,C++,Python source files.                               #

   329

    #                                                                                #

   330

    #   Usage: mki18n {OPTION} [appDirPath]                                          #

   331

    #                                                                                #

   332

    #   Options:                                                                     #

   333

    #     -e               : When -m is used, forces English .mo file creation       #

   334

    #     -h               : prints this help                                        #

   335

    #     -m               : make MO from existing PO files                          #

   336

    #     -p               : make PO, update PO files: Creates a new messages.pot    #

   337

    #                        file. Creates a dom_xx.po.new for every existing        #

   338

    #                        language specific .po file. ('xx' stands for the ISO639 #

   339

    #                        two-letter language code and 'dom' stands for the       #

   340

    #                        application domain name).  mki18n requires that you     #

   341

    #                        write a 'app.fil' file  which contains the list of all  #

   342

    #                        source code to parse.                                   #

   343

    #     -v               : verbose (prints comments while running)                 #

   344

    #     --domain=appName : specifies the application domain name.  By default      #

   345

    #                        the directory name is used.                             #

   346

    #     --moTarget=dir : specifies the directory where .mo files are stored.       #

   347

    #                      If not specified, the target is './locale'                #

   348

    #                                                                                #

   349

    #   You must specify one of the -p or -m option to perform the work.  You can    #

   350

    #   specify the path of the target application.  If you leave it out mki18n      #

   351

    #   will use the current directory as the application main directory.            #

   352

    #                                                                                #

   353

    ##################################################################################""")

   354

    if errorMsg:

   355

        print("\n   ERROR: %s" % errorMsg)

   356

   357

   358

def fileBaseOf(filename, withPath=0):

   359

    """fileBaseOf(filename,withPath) ---> string

   360

   361

    Return base name of filename.  The returned string never includes the extension.

   362

    Use os.path.basename() to return the basename with the extension.  The

   363

    second argument is optional.  If specified and if set to 'true' (non zero)

   364

    the string returned contains the full path of the file name.  Otherwise the

   365

    path is excluded.

   366

   367

    [Example]

   368

    >>> fn = 'd:/dev/telepath/tvapp/code/test.html'

   369

    >>> fileBaseOf(fn)

   370

    'test'

   371

    >>> fileBaseOf(fn)

   372

    'test'

   373

    >>> fileBaseOf(fn,1)

   374

    'd:/dev/telepath/tvapp/code/test'

   375

    >>> fileBaseOf(fn,0)

   376

    'test'

   377

    >>> fn = 'abcdef'

   378

    >>> fileBaseOf(fn)

   379

    'abcdef'

   380

    >>> fileBaseOf(fn,1)

   381

    'abcdef'

   382

    >>> fn = "abcdef."

   383

    >>> fileBaseOf(fn)

   384

    'abcdef'

   385

    >>> fileBaseOf(fn,1)

   386

    'abcdef'

   387

    """

   388

    pos = filename.rfind('.')

   389

    if pos > 0:

   390

        filename = filename[:pos]

   391

    if withPath:

   392

        return filename

   393

    else:

   394

        return os.path.basename(filename)

   395

   396

   397

def mkdir(directory):

   398

    """Create a directory (and possibly the entire tree).

   399

   400

    The os.mkdir() will fail to create a directory if one of the

   401

    directory in the specified path does not exist.  mkdir()

   402

    solves this problem.  It creates every intermediate directory

   403

    required to create the final path. Under Unix, the function

   404

    only supports forward slash separator, but under Windows and MacOS

   405

    the function supports the forward slash and the OS separator (backslash

   406

    under windows).

   407

    """

   408

   409

    # translate the path separators

   410

    directory = unixpath(directory)

   411

    # build a list of all directory elements

   412

    aList = [x for x in directory.split('/') if len(x) > 0]

   413

    theLen = len(aList)

   414

    # if the first element is a Windows-style disk drive

   415

    # concatenate it with the first directory

   416

    if aList[0].endswith(':'):

   417

        if theLen > 1:

   418

            aList[1] = aList[0] + '/' + aList[1]

   419

            del aList[0]

   420

            theLen -= 1

   421

    # if the original directory starts at root,

   422

    # make sure the first element of the list

   423

    # starts at root too

   424

    if directory[0] == '/':

   425

        aList[0] = '/' + aList[0]

   426

    # Now iterate through the list, check if the

   427

    # directory exists and if not create it

   428

    theDir = ''

   429

    for i in range(theLen):

   430

        theDir += aList[i]

   431

        if not os.path.exists(theDir):

   432

            os.mkdir(theDir)

   433

        theDir += '/'

   434

   435

   436

def unixpath(thePath):

   437

    r"""Return a path name that contains Unix separator.

   438

   439

    [Example]

   440

    >>> unixpath(r"d:\test")

   441

    'd:/test'

   442

    >>> unixpath("d:/test/file.txt")

   443

    'd:/test/file.txt'

   444

    >>>

   445

    """

   446

    thePath = os.path.normpath(thePath)

   447

    if os.sep == '/':

   448

        return thePath

   449

    else:

   450

        return thePath.replace(os.sep, '/')

   451

   452

   453

# -----------------------------------------------------------------------------

   454

   455

# S c r i p t   e x e c u t i o n               -- Runs when invoked from the command line --

   456

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

   457

#

   458

if __name__ == "__main__":

   459

    # command line parsing

   460

    import getopt    # pylint: disable=wrong-import-order,wrong-import-position

   461

    argc = len(sys.argv)

   462

    if argc == 1:

   463

        printUsage('Missing argument: specify at least one of -m or -p (or both).')

   464

        sys.exit(1)

   465

    # If there is some arguments, parse the command line

   466

    validOptions = "ehmpv"

   467

    validLongOptions = ['domain=', 'moTarget=']

   468

    option = {}

   469

    option['forceEnglish'] = 0

   470

    option['mo'] = 0

   471

    option['po'] = 0

   472

    option['verbose'] = 0

   473

    option['domain'] = None

   474

    option['moTarget'] = None

   475

    optionKey = {

   476

        '-e':         'forceEnglish',

   477

        '-m':         'mo',

   478

        '-p':         'po',

   479

        '-v':         'verbose',

   480

        '--domain':   'domain',

   481

        '--moTarget': 'moTarget',

   482

    }

   483

    exit_code = 1

   484

    try:

   485

        optionList, pargs = getopt.getopt(sys.argv[1:], validOptions, validLongOptions)

   486

    except getopt.GetoptError as e:

   487

        printUsage(e)

   488

        sys.exit(1)

   489

    for (opt, val) in optionList: