Add tools to the repository.

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17 0039d316-1c4b-4281-b951-d872f2087c98

tools/channel_changer/channel_changer.cc

@@ -0,0 +1,245 @@
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <Windowsx.h>
+
+#include "base/registry.h"
+
+#include "Resource.h"
+
+// This enum needs to be in sync with the strings below.
+enum Branch {
+  UNKNOWN_BRANCH = 0,
+  DEV_BRANCH,
+  BETA_BRANCH,
+};
+
+// This vector of strings needs to be in sync with the Branch enum above.
+static const wchar_t* const kBranchStrings[] = {
+  L"",
+  L"1.1-dev",
+  L"1.1-beta"
+};
+
+// This vector of strings needs to be in sync with the Branch enum above.
+static const wchar_t* const kBranchStringsReadable[] = {
+  L"",
+  L"Dev",
+  L"Beta"
+};
+
+// The root key for Google Update.
+static const HKEY kGoogleUpdateRoot = HKEY_CURRENT_USER;
+
+// The Google Update key to read to find out which branch you are on.
+static const wchar_t* const kGoogleUpdateKey =
+    L"Software\\Google\\Update\\ClientState\\"
+    L"{8A69D345-D564-463C-AFF1-A69D9E530F96}";
+
+// The Google Update value that defines which branch you are on.
+static const wchar_t* const kBranchKey = L"ap";
+
+// The suffix Google Update sometimes adds to the channel name (channel names
+// are defined in kBranchStrings), indicating that a full install is needed. We 
+// strip this out (if present) for the purpose of determining which channel you
+// are on.
+static const wchar_t* const kChannelSuffix = L"-full";
+
+// The icon to use.
+static HICON dlg_icon = NULL;
+
+void SetMainLabel(HWND dialog, Branch branch) {
+  std::wstring main_label = L"You are currently on ";
+  if (branch == DEV_BRANCH || branch == BETA_BRANCH)
+    main_label += std::wstring(L"the ") + kBranchStringsReadable[branch] +
+                  std::wstring(L" channel");
+  else
+    main_label += L"NO UPDATE CHANNEL";
+
+  main_label += L". Choose a different channel and click Update, "
+                L"or click Close to stay on this channel.";
+
+  SetWindowText(GetDlgItem(dialog, IDC_LABEL_MAIN), main_label.c_str());
+}
+
+void OnInitDialog(HWND dialog) {
+  SendMessage(dialog, WM_SETICON, (WPARAM) false, (LPARAM) dlg_icon);
+
+  std::wstring branch_string;
+  RegKey google_update(kGoogleUpdateRoot, kGoogleUpdateKey, KEY_READ);
+  if (google_update.Valid() &&
+      !google_update.ReadValue(kBranchKey, &branch_string)) {
+    // If the 'ap' value is missing, we create it, unless the key is missing.
+    RegKey write_default(kGoogleUpdateRoot, kGoogleUpdateKey, KEY_WRITE);
+    branch_string = kBranchStrings[BETA_BRANCH];
+    if (!write_default.WriteValue(kBranchKey, branch_string.c_str()))
+      branch_string = L"";  // Error, show disabled UI.
+  }
+
+  // We look for '1.1-beta' or '1.1-dev', but Google Update might have added
+  // '-full' to the channel name, which we need to strip out to determine what
+  // channel you are on.
+  std::wstring suffix = kChannelSuffix;
+  if (branch_string.length() > suffix.length()) {
+    size_t index = branch_string.rfind(suffix);
+    if (index != std::wstring::npos &&
+        index == branch_string.length() - suffix.length()) {
+      branch_string = branch_string.substr(0, index);
+    }
+  }
+
+  Branch branch = UNKNOWN_BRANCH;
+  if (branch_string == kBranchStrings[DEV_BRANCH]) {
+    branch = DEV_BRANCH;
+  } else if (branch_string == kBranchStrings[BETA_BRANCH]) {
+    branch = BETA_BRANCH;
+  } else {
+    // Hide the controls we can't use.
+    EnableWindow(GetDlgItem(dialog, IDOK), false);
+    EnableWindow(GetDlgItem(dialog, IDC_STABLE), false);
+    EnableWindow(GetDlgItem(dialog, IDC_CUTTING_EDGE), false);
+
+    MessageBox(dialog, L"KEY NOT FOUND\n\nChrome is not installed, or is not "
+                       L"using GoogleUpdate for updates.",
+                       L"Chrome Channel Changer",
+                       MB_ICONEXCLAMATION | MB_OK);
+  }
+
+  SetMainLabel(dialog, branch);
+
+  CheckDlgButton(dialog, IDC_STABLE,
+                 branch == BETA_BRANCH ? BST_CHECKED : BST_UNCHECKED);
+  CheckDlgButton(dialog, IDC_CUTTING_EDGE,
+                 branch == DEV_BRANCH ? BST_CHECKED : BST_UNCHECKED);
+}
+
+INT_PTR OnCtlColorStatic(HWND dialog, WPARAM wparam, LPARAM lparam) {
+  HDC hdc = reinterpret_cast<HDC>(wparam);
+  HWND control_wnd = reinterpret_cast<HWND>(lparam);
+
+  if (GetDlgItem(dialog, IDC_STABLE) == control_wnd ||
+      GetDlgItem(dialog, IDC_CUTTING_EDGE) == control_wnd ||
+      GetDlgItem(dialog, IDC_LABEL_MAIN) == control_wnd ||
+      GetDlgItem(dialog, IDC_SECONDARY_LABEL) == control_wnd) {
+    SetBkMode(hdc, TRANSPARENT);
+    SetTextColor(hdc, RGB(0, 0, 0));
+    return reinterpret_cast<INT_PTR>(GetSysColorBrush(COLOR_WINDOW));
+  }
+
+  return static_cast<INT_PTR>(FALSE);
+}
+
+void SaveChanges(HWND dialog) {
+  Branch branch = UNKNOWN_BRANCH;
+  if (IsDlgButtonChecked(dialog, IDC_STABLE))
+    branch = BETA_BRANCH;
+  else if (IsDlgButtonChecked(dialog, IDC_CUTTING_EDGE))
+    branch = DEV_BRANCH;
+
+  if (branch != UNKNOWN_BRANCH) {
+    RegKey google_update(kGoogleUpdateRoot, kGoogleUpdateKey, KEY_WRITE);
+    if (!google_update.WriteValue(kBranchKey, kBranchStrings[branch])) {
+      MessageBox(dialog, L"Unable to change value, please make sure you\n"
+                         L"have permission to change registry keys under HKLM",
+                         L"Unable to update branch info", MB_OK);
+    } else {
+      std::wstring save_msg = L"Your changes have been saved.\nYou are now "
+                              L"on the " +
+                              std::wstring(kBranchStringsReadable[branch]) +
+                              L" branch.";
+      MessageBox(dialog, save_msg.c_str(), L"Changes were saved", MB_OK);
+
+      SetMainLabel(dialog, branch);
+    }
+  }
+}
+
+INT_PTR CALLBACK DialogWndProc(HWND dialog,
+                               UINT message_id,
+                               WPARAM wparam,
+                               LPARAM lparam) {
+  UNREFERENCED_PARAMETER(lparam);
+
+  switch (message_id) {
+    case WM_INITDIALOG:
+      OnInitDialog(dialog);
+      return static_cast<INT_PTR>(TRUE);
+    case WM_CTLCOLORSTATIC:
+      return OnCtlColorStatic(dialog, wparam, lparam);
+    case WM_COMMAND:
+      // If the user presses the OK button.
+      if (LOWORD(wparam) == IDOK) {
+        SaveChanges(dialog);
+        return static_cast<INT_PTR>(TRUE);
+      }
+      // If the user presses the Cancel button.
+      if (LOWORD(wparam) == IDCANCEL) {
+        ::EndDialog(dialog, LOWORD(wparam));
+        return static_cast<INT_PTR>(TRUE);
+      }
+      break;
+    case WM_ERASEBKGND:
+      PAINTSTRUCT paint;
+      HDC hdc = BeginPaint(dialog, &paint);
+      if (!hdc)
+        return static_cast<INT_PTR>(FALSE);  // We didn't handle it.
+
+      // Fill the background with White.
+      HBRUSH brush = GetStockBrush(WHITE_BRUSH);
+      HGDIOBJ old_brush = SelectObject(hdc, brush);
+      RECT rc;
+      GetClientRect(dialog, &rc);
+      FillRect(hdc, &rc, brush);
+
+      // Clean up.
+      SelectObject(hdc, old_brush);
+      EndPaint(dialog, &paint);
+      return static_cast<INT_PTR>(TRUE);
+  }
+
+  return static_cast<INT_PTR>(FALSE);
+}
+
+int APIENTRY _tWinMain(HINSTANCE instance,
+                       HINSTANCE previous_instance,
+                       LPTSTR    cmd_line,
+                       int       cmd_show) {
+  UNREFERENCED_PARAMETER(previous_instance);
+  UNREFERENCED_PARAMETER(cmd_line);
+  UNREFERENCED_PARAMETER(cmd_show);
+
+  dlg_icon = ::LoadIcon(instance, MAKEINTRESOURCE(IDI_BRANCH_SWITCHER));
+
+  ::DialogBox(instance,
+              MAKEINTRESOURCE(IDD_MAIN_DIALOG),
+              GetDesktopWindow(),
+              DialogWndProc);
+
+  return TRUE;
+}

tools/channel_changer/channel_changer.rc

@@ -0,0 +1,124 @@
+// Microsoft Visual C++ generated resource script.
+//
+#include "resource.h"
+
+#define APSTUDIO_READONLY_SYMBOLS
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 2 resource.
+//
+#define APSTUDIO_HIDDEN_SYMBOLS
+#include "windows.h"
+#undef APSTUDIO_HIDDEN_SYMBOLS
+
+/////////////////////////////////////////////////////////////////////////////
+#undef APSTUDIO_READONLY_SYMBOLS
+
+/////////////////////////////////////////////////////////////////////////////
+// English (U.S.) resources
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+#ifdef _WIN32
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+#pragma code_page(1252)
+#endif //_WIN32
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Icon
+//
+
+// Icon with lowest ID value placed first to ensure application icon
+// remains consistent on all systems.
+IDI_BRANCH_SWITCHER     ICON                    "channel_changer.ico"
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Dialog
+//
+
+IDD_MAIN_DIALOG DIALOGEX 0, 0, 419, 147
+STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "Chrome Channel Switcher v1.1"
+FONT 8, "MS Shell Dlg", 400, 0, 0x1
+BEGIN
+    DEFPUSHBUTTON   "Update",IDOK,156,127,50,14
+    PUSHBUTTON      "Close",IDCANCEL,212,127,50,14
+    LTEXT           "This label will be populated programmatically.",IDC_LABEL_MAIN,47,13,322,18
+    CONTROL         "Beta: More stable releases, less frequently. You won't get bug fixes and new features as quickly, but what you get will have had more testing.",IDC_STABLE,
+                    "Button",BS_AUTORADIOBUTTON | BS_MULTILINE,47,39,340,19
+    CONTROL         "Dev: Get the latest features and bugfixes every week. You'll be helping us improve, iterate, and make Chrome great faster. We need Dev users, and we'll give your feedback and issues higher priority.",IDC_CUTTING_EDGE,
+                    "Button",BS_AUTORADIOBUTTON | BS_MULTILINE,47,63,355,21
+    CONTROL         129,IDC_IMAGEVIEW,"Static",SS_BITMAP,7,7,35,23
+    LTEXT           "Note: If you switch from Dev to Beta, you'll stay on the current Chrome version until the Beta channel gets a later update. Changing channels does not change the version of Chrome you are currently using.",IDC_SECONDARY_LABEL,47,97,340,21
+END
+
+
+#ifdef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// TEXTINCLUDE
+//
+
+1 TEXTINCLUDE 
+BEGIN
+    "resource.h\0"
+END
+
+2 TEXTINCLUDE 
+BEGIN
+    "#define APSTUDIO_HIDDEN_SYMBOLS\r\n"
+    "#include ""windows.h""\r\n"
+    "#undef APSTUDIO_HIDDEN_SYMBOLS\r\n"
+    "\0"
+END
+
+3 TEXTINCLUDE 
+BEGIN
+    "\r\n"
+    "\0"
+END
+
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// DESIGNINFO
+//
+
+#ifdef APSTUDIO_INVOKED
+GUIDELINES DESIGNINFO 
+BEGIN
+    IDD_MAIN_DIALOG, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 412
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 140
+    END
+END
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Bitmap
+//
+
+IDB_PICTURE             BITMAP                  "glasses.bmp"
+#endif    // English (U.S.) resources
+/////////////////////////////////////////////////////////////////////////////
+
+
+
+#ifndef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 3 resource.
+//
+
+
+/////////////////////////////////////////////////////////////////////////////
+#endif    // not APSTUDIO_INVOKED
+

tools/channel_changer/channel_changer.sln

@@ -0,0 +1,20 @@
+
+Microsoft Visual Studio Solution File, Format Version 9.00
+# Visual Studio 2005
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel_changer", "channel_changer.vcproj", "{93D9CC27-A861-4430-A08B-AEB2FC223D78}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Release|Win32 = Release|Win32
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{93D9CC27-A861-4430-A08B-AEB2FC223D78}.Debug|Win32.ActiveCfg = Debug|Win32
+		{93D9CC27-A861-4430-A08B-AEB2FC223D78}.Debug|Win32.Build.0 = Debug|Win32
+		{93D9CC27-A861-4430-A08B-AEB2FC223D78}.Release|Win32.ActiveCfg = Release|Win32
+		{93D9CC27-A861-4430-A08B-AEB2FC223D78}.Release|Win32.Build.0 = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

tools/channel_changer/channel_changer.vcproj

@@ -0,0 +1,237 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="8.00"
+	Name="channel_changer"
+	ProjectGUID="{93D9CC27-A861-4430-A08B-AEB2FC223D78}"
+	RootNamespace="channel_changer"
+	Keyword="Win32Proj"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			UseOfATL="1"
+			CharacterSet="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="../.."
+				PreprocessorDefinitions="WIN32;_DEBUG;_WINDOWS"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				SubSystem="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCWebDeploymentTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			UseOfATL="1"
+			CharacterSet="1"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				AdditionalIncludeDirectories="../.."
+				PreprocessorDefinitions="WIN32;NDEBUG;_WINDOWS"
+				RuntimeLibrary="2"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				SubSystem="2"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCWebDeploymentTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath=".\channel_changer.cc"
+				>
+			</File>
+			<File
+				RelativePath="..\..\base\registry.cc"
+				>
+				<FileConfiguration
+					Name="Debug|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="..\..\base\registry.h"
+				>
+			</File>
+			<File
+				RelativePath=".\Resource.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+			>
+			<File
+				RelativePath=".\channel_changer.ico"
+				>
+			</File>
+			<File
+				RelativePath=".\channel_changer.rc"
+				>
+			</File>
+			<File
+				RelativePath=".\glasses.bmp"
+				>
+			</File>
+			<File
+				RelativePath=".\switcher_icon.bmp"
+				>
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

tools/channel_changer/resource.h

@@ -0,0 +1,24 @@
+//{{NO_DEPENDENCIES}}
+// Microsoft Visual C++ generated include file.
+// Used by channel_changer.rc
+//
+#define IDI_BRANCH_SWITCHER             101
+#define IDD_MAIN_DIALOG                 128
+#define IDB_PICTURE                     129
+#define IDC_IMAGEVIEW                   130
+#define IDC_LABEL_MAIN                  1000
+#define IDC_STABLE                      1001
+#define IDC_CUTTING_EDGE                1002
+#define IDC_SECONDARY_LABEL             1003
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NO_MFC                     1
+#define _APS_NEXT_RESOURCE_VALUE        131
+#define _APS_NEXT_COMMAND_VALUE         32771
+#define _APS_NEXT_CONTROL_VALUE         1004
+#define _APS_NEXT_SYMED_VALUE           102
+#endif
+#endif

tools/grit/README

@@ -0,0 +1,2 @@
+GRIT (Google Resource and Internationalization Tool) is a tool for Windows
+projects to manage resources and simplify the localization workflow.

tools/grit/build/grit_resource_file.bat

@@ -0,0 +1,15 @@
+:: Batch file run as build command for .grd files
+:: The custom build rule is set to expect (inputfile).h and (inputfile).rc
+:: our grd files must generate files with the same basename.
+@echo off
+
+setlocal
+
+set InFile=%~1
+set SolutionDir=%~2
+set InputDir=%~3
+
+:: Use GNU tools
+call %SolutionDir%\..\third_party\gnu\setup_env.bat
+
+%SolutionDir%\..\third_party\python_24\python.exe %SolutionDir%\..\tools\grit\grit.py -i %InFile% build -o %InputDir%

tools/grit/build/grit_resource_file.rules

@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="utf-8"?>
+<VisualStudioToolFile
+	Name="GRIT resource data file"
+	Version="8.00"
+	>
+	<Rules>
+		<CustomBuildRule
+			Name="GRIT Generated Resources"
+			DisplayName="GRIT Generated Resources"
+			CommandLine="$(SolutionDir)..\tools\grit\build\grit_resource_file.bat [inputs] &quot;$(SolutionDir)&quot; &quot;$(IntDir)&quot;"
+			Outputs="$(IntDir)\$(InputName).h;
+			  $(IntDir)\$(InputName)_ar.rc;
+			  $(IntDir)\$(InputName)_bg.rc;
+			  $(IntDir)\$(InputName)_ca.rc;
+			  $(IntDir)\$(InputName)_cs.rc;
+			  $(IntDir)\$(InputName)_da.rc;
+			  $(IntDir)\$(InputName)_de.rc;
+			  $(IntDir)\$(InputName)_el.rc;
+			  $(IntDir)\$(InputName)_en-GB.rc;
+			  $(IntDir)\$(InputName)_en-US.rc;
+			  $(IntDir)\$(InputName)_es.rc;
+			  $(IntDir)\$(InputName)_es-419.rc;
+			  $(IntDir)\$(InputName)_et.rc;
+			  $(IntDir)\$(InputName)_fi.rc;
+			  $(IntDir)\$(InputName)_fil.rc;
+			  $(IntDir)\$(InputName)_fr.rc;
+			  $(IntDir)\$(InputName)_he.rc;
+			  $(IntDir)\$(InputName)_hi.rc;
+			  $(IntDir)\$(InputName)_hr.rc;
+			  $(IntDir)\$(InputName)_hu.rc;
+			  $(IntDir)\$(InputName)_id.rc;
+			  $(IntDir)\$(InputName)_it.rc;
+			  $(IntDir)\$(InputName)_ja.rc;
+			  $(IntDir)\$(InputName)_ko.rc;
+			  $(IntDir)\$(InputName)_lt.rc;
+			  $(IntDir)\$(InputName)_lv.rc;
+			  $(IntDir)\$(InputName)_nl.rc;
+			  $(IntDir)\$(InputName)_nb.rc;
+			  $(IntDir)\$(InputName)_pl.rc;
+			  $(IntDir)\$(InputName)_pt-BR.rc;
+			  $(IntDir)\$(InputName)_pt-PT.rc;
+			  $(IntDir)\$(InputName)_ro.rc;
+			  $(IntDir)\$(InputName)_ru.rc;
+			  $(IntDir)\$(InputName)_sk.rc;
+			  $(IntDir)\$(InputName)_sl.rc;
+			  $(IntDir)\$(InputName)_sr.rc;
+			  $(IntDir)\$(InputName)_sv.rc;
+			  $(IntDir)\$(InputName)_th.rc;
+			  $(IntDir)\$(InputName)_tr.rc;
+			  $(IntDir)\$(InputName)_uk.rc;
+			  $(IntDir)\$(InputName)_vi.rc;
+			  $(IntDir)\$(InputName)_zh-CN.rc;
+			  $(IntDir)\$(InputName)_zh-TW.rc;"
+			AdditionalDependencies="$(SolutionDir)..\tools\grit\build\grit_resource_file.bat;$(SolutionDir)..\tools\grit\grit.py;
+			  resources\$(InputName)_ar.xtb;
+			  resources\$(InputName)_bg.xtb;
+			  resources\$(InputName)_ca.xtb;
+			  resources\$(InputName)_cs.xtb;
+			  resources\$(InputName)_da.xtb;
+			  resources\$(InputName)_de.xtb;
+			  resources\$(InputName)_el.xtb;
+			  resources\$(InputName)_en-GB.xtb;
+			  resources\$(InputName)_es.xtb;
+			  resources\$(InputName)_es-419.xtb;
+			  resources\$(InputName)_et.xtb;
+			  resources\$(InputName)_fi.xtb;
+			  resources\$(InputName)_fil.xtb;
+			  resources\$(InputName)_fr.xtb;
+			  resources\$(InputName)_he.xtb;
+			  resources\$(InputName)_hi.xtb;
+			  resources\$(InputName)_hr.xtb;
+			  resources\$(InputName)_hu.xtb;
+			  resources\$(InputName)_id.xtb;
+			  resources\$(InputName)_it.xtb;
+			  resources\$(InputName)_ja.xtb;
+			  resources\$(InputName)_ko.xtb;
+			  resources\$(InputName)_lt.xtb;
+			  resources\$(InputName)_lv.xtb;
+			  resources\$(InputName)_nl.xtb;
+			  resources\$(InputName)_no.xtb;
+			  resources\$(InputName)_pl.xtb;
+			  resources\$(InputName)_pt-BR.xtb;
+			  resources\$(InputName)_pt-PT.xtb;
+			  resources\$(InputName)_ro.xtb;
+			  resources\$(InputName)_ru.xtb;
+			  resources\$(InputName)_sk.xtb;
+			  resources\$(InputName)_sl.xtb;
+			  resources\$(InputName)_sr.xtb;
+			  resources\$(InputName)_sv.xtb;
+			  resources\$(InputName)_th.xtb;
+			  resources\$(InputName)_tr.xtb;
+			  resources\$(InputName)_uk.xtb;
+			  resources\$(InputName)_vi.xtb;
+			  resources\$(InputName)_zh-CN.xtb;
+			  resources\$(InputName)_zh-TW.xtb;"
+			FileExtensions="*.grd"
+			ExecutionDescription="Generating resources..."
+			>
+			<Properties>
+			</Properties>
+		</CustomBuildRule>
+	</Rules>
+</VisualStudioToolFile>

tools/grit/grit.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Bootstrapping for GRIT.
+'''
+
+import sys
+
+import grit.grit_runner
+
+
+if __name__ == '__main__':
+  grit.grit_runner.Main(sys.argv[1:])

tools/grit/grit/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Package 'grit'
+'''
+
+pass

tools/grit/grit/clique.py

@@ -0,0 +1,467 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Collections of messages and their translations, called cliques.  Also
+collections of cliques (uber-cliques).
+'''
+
+import types
+
+from grit import constants
+from grit import exception
+from grit import pseudo
+from grit import tclib
+
+
+class UberClique(object):
+  '''A factory (NOT a singleton factory) for making cliques.  It has several
+  methods for working with the cliques created using the factory.
+  '''
+  
+  def __init__(self):
+    # A map from message ID to list of cliques whose source messages have
+    # that ID.  This will contain all cliques created using this factory.
+    # Different messages can have the same ID because they have the
+    # same translateable portion and placeholder names, but occur in different
+    # places in the resource tree.
+    self.cliques_ = {}
+    
+    # A map of clique IDs to list of languages to indicate translations where we
+    # fell back to English.
+    self.fallback_translations_ = {}
+    
+    # A map of clique IDs to list of languages to indicate missing translations.
+    self.missing_translations_ = {}
+  
+  def _AddMissingTranslation(self, lang, clique, is_error):
+    tl = self.fallback_translations_
+    if is_error:
+      tl = self.missing_translations_
+    id = clique.GetId()
+    if id not in tl:
+      tl[id] = {}
+    if lang not in tl[id]:
+      tl[id][lang] = 1
+  
+  def HasMissingTranslations(self):
+    return len(self.missing_translations_) > 0
+  
+  def MissingTranslationsReport(self):
+    '''Returns a string suitable for printing to report missing
+    and fallback translations to the user.
+    '''
+    def ReportTranslation(clique, langs):
+      text = clique.GetMessage().GetPresentableContent()
+      extract = text[0:40]
+      ellipsis = ''
+      if len(text) > 40:
+        ellipsis = '...'
+      langs_extract = langs[0:6]
+      describe_langs = ','.join(langs_extract)
+      if len(langs) > 6:
+        describe_langs += " and %d more" % (len(langs) - 6)
+      return "  %s \"%s%s\" %s" % (clique.GetId(), extract, ellipsis,
+                                   describe_langs)
+    lines = []
+    if len(self.fallback_translations_):
+      lines.append(
+        "WARNING: Fell back to English for the following translations:")
+      for (id, langs) in self.fallback_translations_.items():
+        lines.append(ReportTranslation(self.cliques_[id][0], langs.keys()))
+    if len(self.missing_translations_):
+      lines.append("ERROR: The following translations are MISSING:")
+      for (id, langs) in self.missing_translations_.items():
+        lines.append(ReportTranslation(self.cliques_[id][0], langs.keys()))
+    return '\n'.join(lines)
+
+  def MakeClique(self, message, translateable=True):
+    '''Create a new clique initialized  with a message.
+    
+    Args:
+      message: tclib.Message()
+      translateable: True | False
+    '''
+    clique = MessageClique(self, message, translateable)
+    
+    # Enable others to find this clique by its message ID
+    if message.GetId() in self.cliques_:
+      presentable_text = clique.GetMessage().GetPresentableContent()
+      for c in self.cliques_[message.GetId()]:
+        assert c.GetMessage().GetPresentableContent() == presentable_text
+      self.cliques_[message.GetId()].append(clique)
+    else:
+      self.cliques_[message.GetId()] = [clique]
+    
+    return clique
+
+  def FindCliqueAndAddTranslation(self, translation, language):
+    '''Adds the specified translation to the clique with the source message
+    it is a translation of.
+    
+    Args:
+      translation: tclib.Translation()
+      language: 'en' | 'fr' ...
+    
+    Return:
+      True if the source message was found, otherwise false.
+    '''
+    if translation.GetId() in self.cliques_:
+      for clique in self.cliques_[translation.GetId()]:
+        clique.AddTranslation(translation, language)
+      return True
+    else:
+      return False
+  
+  def BestClique(self, id):
+    '''Returns the "best" clique from a list of cliques.  All the cliques
+    must have the same ID.  The "best" clique is chosen in the following
+    order of preference:
+    - The first clique that has a non-ID-based description
+    - If no such clique found, one of the cliques with an ID-based description
+    - Otherwise an arbitrary clique
+    '''
+    clique_list = self.cliques_[id]
+    clique_to_ret = None
+    for clique in clique_list:
+      if not clique_to_ret:
+        clique_to_ret = clique
+        
+      description = clique.GetMessage().GetDescription()
+      if description and len(description) > 0:
+        clique_to_ret = clique
+        if not description.startswith('ID:'):
+          break  # this is the preferred case so we exit right away
+    return clique_to_ret
+  
+  def BestCliquePerId(self):
+    '''Iterates over the list of all cliques and returns the best clique for
+    each ID.  This will be the first clique with a source message that has a
+    non-empty description, or an arbitrary clique if none of them has a
+    description.
+    '''
+    for id in self.cliques_:
+      yield self.BestClique(id)
+  
+  def BestCliqueByOriginalText(self, text, meaning):
+    '''Finds the "best" (as in BestClique()) clique that has original text
+    'text' and meaning 'meaning'.  Returns None if there is no such clique.
+    '''
+    # If needed, this can be optimized by maintaining a map of
+    # fingerprints of original text+meaning to cliques.
+    for c in self.BestCliquePerId():
+      msg = c.GetMessage()
+      if msg.GetRealContent() == text and msg.GetMeaning() == meaning:
+        return msg
+    return None
+
+  def AllMessageIds(self):
+    '''Returns a list of all defined message IDs.
+    '''
+    return self.cliques_.keys()
+  
+  def AllCliques(self):
+    '''Iterates over all cliques.  Note that this can return multiple cliques
+    with the same ID.
+    '''
+    for cliques in self.cliques_.values():
+      for c in cliques:
+        yield c
+  
+  def GenerateXtbParserCallback(self, lang, debug=False):
+    '''Creates a callback function as required by grit.xtb_reader.Parse().
+    This callback will create Translation objects for each message from
+    the XTB that exists in this uberclique, and add them as translations for
+    the relevant cliques.  The callback will add translations to the language
+    specified by 'lang'
+    
+    Args:
+      lang: 'fr'
+      debug: True | False
+    '''
+    def Callback(id, structure):
+      if id not in self.cliques_:
+        if debug: print "Ignoring translation #%s" % id
+        return
+      
+      if debug: print "Adding translation #%s" % id
+      
+      # We fetch placeholder information from the original message (the XTB file
+      # only contains placeholder names).
+      original_msg = self.BestClique(id).GetMessage()
+      
+      translation = tclib.Translation(id=id)
+      for is_ph,text in structure:
+        if not is_ph:
+          translation.AppendText(text)
+        else:
+          found_placeholder = False
+          for ph in original_msg.GetPlaceholders():
+            if ph.GetPresentation() == text:
+              translation.AppendPlaceholder(tclib.Placeholder(
+                ph.GetPresentation(), ph.GetOriginal(), ph.GetExample()))
+              found_placeholder = True
+              break
+          if not found_placeholder:
+            raise exception.MismatchingPlaceholders(
+              'Translation for message ID %s had <ph name="%s%/>, no match\n'
+              'in original message' % (id, text))
+      self.FindCliqueAndAddTranslation(translation, lang)
+    return Callback
+
+
+class CustomType(object):
+  '''A base class you should implement if you wish to specify a custom type
+  for a message clique (i.e. custom validation and optional modification of
+  translations).'''
+  
+  def Validate(self, message):
+    '''Returns true if the message (a tclib.Message object) is valid,
+    otherwise false.
+    '''
+    raise NotImplementedError()
+  
+  def ValidateAndModify(self, lang, translation):
+    '''Returns true if the translation (a tclib.Translation object) is valid,
+    otherwise false.  The language is also passed in.  This method may modify
+    the translation that is passed in, if it so wishes.
+    '''
+    raise NotImplementedError()
+  
+  def ModifyTextPart(self, lang, text):
+    '''If you call ModifyEachTextPart, it will turn around and call this method
+    for each text part of the translation.  You should return the modified
+    version of the text, or just the original text to not change anything.
+    '''
+    raise NotImplementedError()
+  
+  def ModifyEachTextPart(self, lang, translation):
+    '''Call this to easily modify one or more of the textual parts of a
+    translation.  It will call ModifyTextPart for each part of the
+    translation.
+    '''
+    contents = translation.GetContent()
+    for ix in range(len(contents)):
+      if (isinstance(contents[ix], types.StringTypes)):
+        contents[ix] = self.ModifyTextPart(lang, contents[ix])
+
+
+class OneOffCustomType(CustomType):
+  '''A very simple custom type that performs the validation expressed by
+  the input expression on all languages including the source language.
+  The expression can access the variables 'lang', 'msg' and 'text()' where 'lang'
+  is the language of 'msg', 'msg' is the message or translation being
+  validated and 'text()' returns the real contents of 'msg' (for shorthand).
+  '''
+  def __init__(self, expression):
+    self.expr = expression
+  def Validate(self, message):
+    return self.ValidateAndModify(MessageClique.source_language, message)
+  def ValidateAndModify(self, lang, msg):
+    def text():
+      return msg.GetRealContent()
+    return eval(self.expr, {},
+            {'lang' : lang,
+             'text' : text,
+             'msg' : msg,
+             })
+
+
+class MessageClique(object):
+  '''A message along with all of its translations.  Also code to bring
+  translations together with their original message.'''
+  
+  # change this to the language code of Messages you add to cliques_.
+  # TODO(joi) Actually change this based on the <grit> node's source language
+  source_language = 'en'
+  
+  # A constant translation we use when asked for a translation into the
+  # special language constants.CONSTANT_LANGUAGE.
+  CONSTANT_TRANSLATION = tclib.Translation(text='TTTTTT')
+  
+  def __init__(self, uber_clique, message, translateable=True, custom_type=None):
+    '''Create a new clique initialized with just a message.
+    
+    Args:
+      uber_clique: Our uber-clique (collection of cliques)
+      message: tclib.Message()
+      translateable: True | False
+      custom_type: instance of clique.CustomType interface
+    '''
+    # Our parent
+    self.uber_clique = uber_clique
+    # If not translateable, we only store the original message.
+    self.translateable = translateable
+    # A mapping of language identifiers to tclib.BaseMessage and its
+    # subclasses (i.e. tclib.Message and tclib.Translation).
+    self.clique = { MessageClique.source_language : message }
+    # A list of the "shortcut groups" this clique is
+    # part of.  Within any given shortcut group, no shortcut key (e.g. &J)
+    # must appear more than once in each language for all cliques that
+    # belong to the group.
+    self.shortcut_groups = []
+    # An instance of the CustomType interface, or None.  If this is set, it will
+    # be used to validate the original message and translations thereof, and
+    # will also get a chance to modify translations of the message.
+    self.SetCustomType(custom_type)
+  
+  def GetMessage(self):
+    '''Retrieves the tclib.Message that is the source for this clique.'''
+    return self.clique[MessageClique.source_language]
+  
+  def GetId(self):
+    '''Retrieves the message ID of the messages in this clique.'''
+    return self.GetMessage().GetId()
+  
+  def IsTranslateable(self):
+    return self.translateable
+  
+  def AddToShortcutGroup(self, group):
+    self.shortcut_groups.append(group)
+  
+  def SetCustomType(self, custom_type):
+    '''Makes this clique use custom_type for validating messages and
+    translations, and optionally modifying translations.
+    '''
+    self.custom_type = custom_type
+    if custom_type and not custom_type.Validate(self.GetMessage()):
+      raise exception.InvalidMessage(self.GetMessage().GetRealContent())
+  
+  def MessageForLanguage(self, lang, pseudo_if_no_match=True, fallback_to_english=False):
+    '''Returns the message/translation for the specified language, providing
+    a pseudotranslation if there is no available translation and a pseudo-
+    translation is requested.
+    
+    The translation of any message whatsoever in the special language
+    'x_constant' is the message "TTTTTT".
+    
+    Args:
+      lang: 'en'
+      pseudo_if_no_match: True
+      fallback_to_english: False
+    
+    Return:
+      tclib.BaseMessage
+    '''
+    if not self.translateable:
+      return self.GetMessage()
+    
+    if lang == constants.CONSTANT_LANGUAGE:
+      return self.CONSTANT_TRANSLATION
+    
+    for msglang in self.clique.keys():
+      if lang == msglang:
+        return self.clique[msglang]
+    
+    if fallback_to_english:
+      self.uber_clique._AddMissingTranslation(lang, self, is_error=False)
+      return self.GetMessage()
+    
+    # If we're not supposed to generate pseudotranslations, we add an error
+    # report to a list of errors, then fail at a higher level, so that we
+    # get a list of all messages that are missing translations.
+    if not pseudo_if_no_match:
+      self.uber_clique._AddMissingTranslation(lang, self, is_error=True)
+    
+    return pseudo.PseudoMessage(self.GetMessage())
+
+  def AllMessagesThatMatch(self, lang_re, include_pseudo = True):
+    '''Returns a map of all messages that match 'lang', including the pseudo
+    translation if requested.
+    
+    Args:
+      lang_re: re.compile('fr|en')
+      include_pseudo: True
+    
+    Return:
+      { 'en' : tclib.Message,
+        'fr' : tclib.Translation,
+        pseudo.PSEUDO_LANG : tclib.Translation }
+    '''
+    if not self.translateable:
+      return [self.GetMessage()]
+    
+    matches = {}
+    for msglang in self.clique:
+      if lang_re.match(msglang):
+        matches[msglang] = self.clique[msglang]
+    
+    if include_pseudo:
+      matches[pseudo.PSEUDO_LANG] = pseudo.PseudoMessage(self.GetMessage())
+    
+    return matches
+  
+  def AddTranslation(self, translation, language):
+    '''Add a translation to this clique.  The translation must have the same
+    ID as the message that is the source for this clique.
+    
+    If this clique is not translateable, the function just returns.
+    
+    Args:
+      translation: tclib.Translation()
+      language: 'en'
+    
+    Throws:
+      grit.exception.InvalidTranslation if the translation you're trying to add
+      doesn't have the same message ID as the source message of this clique.
+    '''
+    if not self.translateable:
+      return
+    if translation.GetId() != self.GetId():
+      raise exception.InvalidTranslation(
+        'Msg ID %s, transl ID %s' % (self.GetId(), translation.GetId()))
+    
+    assert not language in self.clique
+    
+    # Because two messages can differ in the original content of their
+    # placeholders yet share the same ID (because they are otherwise the
+    # same), the translation we are getting may have different original
+    # content for placeholders than our message, yet it is still the right
+    # translation for our message (because it is for the same ID).  We must
+    # therefore fetch the original content of placeholders from our original
+    # English message.
+    #
+    # See grit.clique_unittest.MessageCliqueUnittest.testSemiIdenticalCliques
+    # for a concrete explanation of why this is necessary.
+    
+    original = self.MessageForLanguage(self.source_language, False)
+    if len(original.GetPlaceholders()) != len(translation.GetPlaceholders()):
+      print ("ERROR: '%s' translation of message id %s does not match" %
+             (language, translation.GetId()))
+      assert False
+    
+    transl_msg = tclib.Translation(id=self.GetId(),
+                                   text=translation.GetPresentableContent(),
+                                   placeholders=original.GetPlaceholders())
+    
+    if self.custom_type and not self.custom_type.ValidateAndModify(language, transl_msg):
+      print "WARNING: %s translation failed validation: %s" % (
+        language, transl_msg.GetId())
+    
+    self.clique[language] = transl_msg

tools/grit/grit/clique_unittest.py

@@ -0,0 +1,228 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.clique'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import re
+import StringIO
+import unittest
+
+from grit import clique
+from grit import exception
+from grit import pseudo
+from grit import tclib
+from grit import grd_reader
+from grit import util
+
+class MessageCliqueUnittest(unittest.TestCase):
+  def testClique(self):
+    factory = clique.UberClique()
+    msg = tclib.Message(text='Hello USERNAME, how are you?',
+                        placeholders=[
+                          tclib.Placeholder('USERNAME', '%s', 'Joi')])
+    c = factory.MakeClique(msg)
+    
+    self.failUnless(c.GetMessage() == msg)
+    self.failUnless(c.GetId() == msg.GetId())
+    
+    msg_fr = tclib.Translation(text='Bonjour USERNAME, comment ca va?',
+                               id=msg.GetId(), placeholders=[
+                                tclib.Placeholder('USERNAME', '%s', 'Joi')])
+    msg_de = tclib.Translation(text='Guten tag USERNAME, wie geht es dir?',
+                               id=msg.GetId(), placeholders=[
+                                tclib.Placeholder('USERNAME', '%s', 'Joi')])
+    
+    c.AddTranslation(msg_fr, 'fr')
+    factory.FindCliqueAndAddTranslation(msg_de, 'de')
+    
+    # sort() sorts lists in-place and does not return them
+    for lang in ('en', 'fr', 'de'):
+      self.failUnless(lang in c.clique)
+    
+    self.failUnless(c.MessageForLanguage('fr').GetRealContent() ==
+                    msg_fr.GetRealContent())
+
+    try:
+      c.MessageForLanguage('zh-CN', False)
+      self.fail('Should have gotten exception')
+    except:
+      pass
+
+    self.failUnless(c.MessageForLanguage('zh-CN', True) != None)
+    
+    rex = re.compile('fr|de|bingo')
+    self.failUnless(len(c.AllMessagesThatMatch(rex, False)) == 2)
+    self.failUnless(c.AllMessagesThatMatch(rex, True)[pseudo.PSEUDO_LANG] != None)
+  
+  def testBestClique(self):
+    factory = clique.UberClique()
+    factory.MakeClique(tclib.Message(text='Alfur', description='alfaholl'))
+    factory.MakeClique(tclib.Message(text='Alfur', description=''))
+    factory.MakeClique(tclib.Message(text='Vaettur', description=''))
+    factory.MakeClique(tclib.Message(text='Vaettur', description=''))
+    factory.MakeClique(tclib.Message(text='Troll', description=''))
+    factory.MakeClique(tclib.Message(text='Gryla', description='ID: IDS_GRYLA'))
+    factory.MakeClique(tclib.Message(text='Gryla', description='vondakerling'))
+    factory.MakeClique(tclib.Message(text='Leppaludi', description='ID: IDS_LL'))
+    factory.MakeClique(tclib.Message(text='Leppaludi', description=''))
+    
+    count_best_cliques = 0
+    for c in factory.BestCliquePerId():
+      count_best_cliques += 1
+      msg = c.GetMessage()
+      text = msg.GetRealContent()
+      description = msg.GetDescription()
+      if text == 'Alfur':
+        self.failUnless(description == 'alfaholl')
+      elif text == 'Gryla':
+        self.failUnless(description == 'vondakerling')
+      elif text == 'Leppaludi':
+        self.failUnless(description == 'ID: IDS_LL')
+    self.failUnless(count_best_cliques == 5)
+
+  def testAllInUberClique(self):
+    resources = grd_reader.Parse(util.WrapInputStream(
+      StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+  <release seq="3">
+    <messages>
+      <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+        Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+      </message>
+    </messages>
+    <structures>
+      <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="grit/test/data/klonk.rc" />
+      <structure type="tr_html" name="ID_HTML" file="grit/test/data/simple.html" />
+    </structures>
+  </release>
+</grit>''')), util.PathFromRoot('.'))
+    resources.RunGatherers(True)
+    content_list = []
+    for clique_list in resources.UberClique().cliques_.values():
+      for clique in clique_list:
+        content_list.append(clique.GetMessage().GetRealContent())
+    self.failUnless('Hello %s, how are you doing today?' in content_list)
+    self.failUnless('Jack "Black" Daniels' in content_list)
+    self.failUnless('Hello!' in content_list)
+  
+  def testCorrectExceptionIfWrongEncodingOnResourceFile(self):
+    '''This doesn't really belong in this unittest file, but what the heck.'''
+    resources = grd_reader.Parse(util.WrapInputStream(
+      StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+  <release seq="3">
+    <structures>
+      <structure type="dialog" name="IDD_ABOUTBOX" file="grit/test/data/klonk.rc" />
+    </structures>
+  </release>
+</grit>''')), util.PathFromRoot('.'))
+    self.assertRaises(exception.SectionNotFound, resources.RunGatherers, True)
+
+  def testSemiIdenticalCliques(self):
+    messages = [
+      tclib.Message(text='Hello USERNAME',
+                    placeholders=[tclib.Placeholder('USERNAME', '$1', 'Joi')]),
+      tclib.Message(text='Hello USERNAME',
+                    placeholders=[tclib.Placeholder('USERNAME', '%s', 'Joi')]),
+    ]
+    self.failUnless(messages[0].GetId() == messages[1].GetId())
+    
+    # Both of the above would share a translation.
+    translation = tclib.Translation(id=messages[0].GetId(),
+                                    text='Bonjour USERNAME',
+                                    placeholders=[tclib.Placeholder(
+                                      'USERNAME', '$1', 'Joi')])
+    
+    factory = clique.UberClique()
+    cliques = [factory.MakeClique(msg) for msg in messages]
+    
+    for clq in cliques:
+      clq.AddTranslation(translation, 'fr')
+    
+    self.failUnless(cliques[0].MessageForLanguage('fr').GetRealContent() ==
+                    'Bonjour $1')
+    self.failUnless(cliques[1].MessageForLanguage('fr').GetRealContent() ==
+                    'Bonjour %s')
+  
+  def testMissingTranslations(self):
+    messages = [ tclib.Message(text='Hello'), tclib.Message(text='Goodbye') ]
+    factory = clique.UberClique()
+    cliques = [factory.MakeClique(msg) for msg in messages]
+    
+    cliques[1].MessageForLanguage('fr', False, True)
+    
+    self.failUnless(not factory.HasMissingTranslations())
+
+    cliques[0].MessageForLanguage('de', False, False)
+    
+    self.failUnless(factory.HasMissingTranslations())
+    
+    report = factory.MissingTranslationsReport()
+    self.failUnless(report.count('WARNING') == 1)
+    self.failUnless(report.count('8053599568341804890 "Goodbye" fr') == 1)
+    self.failUnless(report.count('ERROR') == 1)
+    self.failUnless(report.count('800120468867715734 "Hello" de') == 1)
+
+  def testCustomTypes(self):
+    factory = clique.UberClique()
+    message = tclib.Message(text='Bingo bongo')
+    c = factory.MakeClique(message)
+    try:
+      c.SetCustomType(DummyCustomType())
+      self.fail()
+    except:
+      pass  # expected case - 'Bingo bongo' does not start with 'jjj'
+    
+    message = tclib.Message(text='jjjBingo bongo')
+    c = factory.MakeClique(message)
+    c.SetCustomType(util.NewClassInstance(
+      'grit.clique_unittest.DummyCustomType', clique.CustomType))
+    translation = tclib.Translation(id=message.GetId(), text='Bilingo bolongo')
+    c.AddTranslation(translation, 'fr')
+    self.failUnless(c.MessageForLanguage('fr').GetRealContent().startswith('jjj'))
+
+
+class DummyCustomType(clique.CustomType):
+  def Validate(self, message):
+    return message.GetRealContent().startswith('jjj')
+  def ValidateAndModify(self, lang, translation):
+    is_ok = self.Validate(translation)
+    self.ModifyEachTextPart(lang, translation)
+  def ModifyTextPart(self, lang, text):
+    return 'jjj%s' % text
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/constants.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Constant definitions for GRIT.
+'''
+
+
+# This is the Icelandic noun meaning "grit" and is used to check that our
+# input files are in the correct encoding.  The middle character gets encoded
+# as two bytes in UTF-8, so this is sufficient to detect incorrect encoding.
+ENCODING_CHECK = u'm\u00f6l'
+
+# A special language, translations into which are always "TTTTTT".
+CONSTANT_LANGUAGE = 'x_constant'
+
+# The Unicode byte-order-marker character (this is the Unicode code point,
+# not the encoding of that character into any particular Unicode encoding).
+BOM = u"\ufeff"

tools/grit/grit/exception.py

@@ -0,0 +1,177 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Exception types for GRIT.
+'''
+
+class Base(Exception):
+  '''A base exception that uses the class's docstring in addition to any
+  user-provided message as the body of the Base.
+  '''
+  def __init__(self, msg=''):
+    if len(msg):
+      if self.__doc__:
+        msg = self.__doc__ + ': ' + msg
+    else:
+      msg = self.__doc__
+    Exception.__init__(self, msg)
+
+
+class Parsing(Base):
+  '''An error occurred parsing a GRD or XTB file.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class UnknownElement(Parsing):
+  '''An unknown node type was encountered.'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class MissingElement(Parsing):
+  '''An expected element was missing.'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class UnexpectedChild(Parsing):
+  '''An unexpected child element was encountered (on a leaf node).'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class UnexpectedAttribute(Parsing):
+  '''The attribute was not expected'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class UnexpectedContent(Parsing):
+  '''This element should not have content'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class MissingMandatoryAttribute(Parsing):
+  '''This element is missing a mandatory attribute'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class MutuallyExclusiveMandatoryAttribute(Parsing):
+  '''This element has 2 mutually exclusive mandatory attributes'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class DuplicateKey(Parsing):
+  '''A duplicate key attribute was found.'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class TooManyExamples(Parsing):
+  '''Only one <ex> element is allowed for each <ph> element.'''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class GotPathExpectedFilenameOnly(Parsing):
+  '''The 'filename' attribute of an <output> node must not be a path, only
+  a filename.
+  '''
+  def __init__(self, msg=''):
+    Parsing.__init__(self, msg)
+
+
+class InvalidMessage(Base):
+  '''The specified message failed validation.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class InvalidTranslation(Base):
+  '''Attempt to add an invalid translation to a clique.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class NoSuchTranslation(Base):
+  '''Requested translation not available'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class NotReady(Base):
+  '''Attempt to use an object before it is ready, or attempt to translate
+  an empty document.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class TooManyPlaceholders(Base):
+  '''Too many placeholders for elements of the same type.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class MismatchingPlaceholders(Base):
+  '''Placeholders do not match.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class InvalidPlaceholderName(Base):
+  '''Placeholder name can only contain A-Z, a-z, 0-9 and underscore.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class BlockTagInTranslateableChunk(Base):
+  '''A block tag was encountered where it wasn't expected.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class SectionNotFound(Base):
+  '''The section you requested was not found in the RC file. Make
+sure the section ID is correct (matches the section's ID in the RC file).
+Also note that you may need to specify the RC file's encoding (using the
+encoding="" attribute) if it is not in the default Windows-1252 encoding.
+  '''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)
+
+
+class IdRangeOverlap(Base):
+  '''ID range overlap.'''
+  def __init__(self, msg=''):
+    Base.__init__(self, msg)

tools/grit/grit/extern/FP.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python2.2
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import md5
+
+"""64-bit fingerprint support for strings.
+
+Usage:
+    from extern import FP
+    print 'Fingerprint is %ld' % FP.FingerPrint('Hello world!')
+"""
+
+
+def UnsignedFingerPrint(str, encoding='utf-8'):
+  """Generate a 64-bit fingerprint by taking the first half of the md5
+  of the string."""
+  hex128 = md5.new(str).hexdigest()
+  int64 = long(hex128[:16], 16)
+  return int64
+
+def FingerPrint(str, encoding='utf-8'):
+  fp = UnsignedFingerPrint(str, encoding=encoding)
+  # interpret fingerprint as signed longs
+  if fp & 0x8000000000000000L:
+    fp = - ((~fp & 0xFFFFFFFFFFFFFFFFL) + 1)
+  return fp
+ 

tools/grit/grit/extern/tclib.py

@@ -0,0 +1,527 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The tclib module contains tools for aggregating, verifying, and storing
+# messages destined for the Translation Console, as well as for reading
+# translations back and outputting them in some desired format.
+#
+# This has been stripped down to include only the functionality needed by grit
+# for creating Windows .rc and .h files.  These are the only parts needed by
+# the Chrome build process.
+
+import exceptions
+
+from grit.extern import FP
+
+# This module assumes that within a bundle no two messages can have the
+# same id unless they're identical.
+
+# The basic classes defined here for external use are Message and Translation,
+# where the former is used for English messages and the latter for
+# translations. These classes have a lot of common functionality, as expressed
+# by the common parent class BaseMessage. Perhaps the most important
+# distinction is that translated text is stored in UTF-8, whereas original text
+# is stored in whatever encoding the client uses (presumably Latin-1).
+
+# --------------------
+# The public interface
+# --------------------
+
+# Generate message id from message text and meaning string (optional),
+# both in utf-8 encoding
+#
+def GenerateMessageId(message, meaning=''):
+  fp = FP.FingerPrint(message)
+  if meaning:
+    # combine the fingerprints of message and meaning
+    fp2 = FP.FingerPrint(meaning)
+    if fp < 0:
+      fp = fp2 + (fp << 1) + 1
+    else:
+      fp = fp2 + (fp << 1)
+  # To avoid negative ids we strip the high-order bit
+  return str(fp & 0x7fffffffffffffffL)
+
+# -------------------------------------------------------------------------
+# The MessageTranslationError class is used to signal tclib-specific errors.
+
+class MessageTranslationError(exceptions.Exception):
+  def __init__(self, args = ''):
+    self.args = args
+
+
+# -----------------------------------------------------------
+# The Placeholder class represents a placeholder in a message.
+
+class Placeholder(object):
+  # String representation
+  def __str__(self):
+    return '%s, "%s", "%s"' % \
+           (self.__presentation, self.__original, self.__example)
+
+  # Getters
+  def GetOriginal(self):
+    return self.__original
+
+  def GetPresentation(self):
+    return self.__presentation
+
+  def GetExample(self):
+    return self.__example
+
+  def __eq__(self, other):
+    return self.EqualTo(other, strict=1, ignore_trailing_spaces=0)
+
+  # Equality test
+  #
+  # ignore_trailing_spaces: TC is using varchar to store the
+  # phrwr fields, as a result of that, the trailing spaces
+  # are removed by MySQL when the strings are stored into TC:-(
+  # ignore_trailing_spaces parameter is used to ignore
+  # trailing spaces during equivalence comparison.
+  #
+  def EqualTo(self, other, strict = 1, ignore_trailing_spaces = 1):
+    if type(other) is not Placeholder:
+      return 0
+    if StringEquals(self.__presentation, other.__presentation,
+                    ignore_trailing_spaces):
+      if not strict or (StringEquals(self.__original, other.__original,
+                                     ignore_trailing_spaces)  and
+                        StringEquals(self.__example, other.__example,
+                                     ignore_trailing_spaces)):
+        return 1
+    return 0
+
+
+# -----------------------------------------------------------------
+# BaseMessage is the common parent class of Message and Translation.
+# It is not meant for direct use.
+
+class BaseMessage(object):
+  # Three types of message construction is supported. If the message text is a
+  # simple string with no dynamic content, you can pass it to the constructor
+  # as the "text" parameter. Otherwise, you can omit "text" and assemble the
+  # message step by step using AppendText() and AppendPlaceholder(). Or, as an
+  # alternative, you can give the constructor the "presentable" version of the
+  # message and a list of placeholders; it will then parse the presentation and
+  # build the message accordingly. For example:
+  # Message(text = "There are NUM_BUGS bugs in your code",
+  #         placeholders = [Placeholder("NUM_BUGS", "%d", "33")],
+  #         description = "Bla bla bla")
+  def __eq__(self, other):
+    # "source encoding" is nonsense, so ignore it
+    return _ObjectEquals(self, other, ['_BaseMessage__source_encoding'])
+
+  def GetName(self):
+    return self.__name
+
+  def GetSourceEncoding(self):
+    return self.__source_encoding
+
+  # Append a placeholder to the message
+  def AppendPlaceholder(self, placeholder):
+    if not isinstance(placeholder, Placeholder):
+      raise MessageTranslationError, ("Invalid message placeholder %s in "
+                                      "message %s" % (placeholder, self.GetId()))
+    # Are there other placeholders with the same presentation?
+    # If so, they need to be the same.
+    for other in self.GetPlaceholders():
+      if placeholder.GetPresentation() == other.GetPresentation():
+        if not placeholder.EqualTo(other):
+          raise MessageTranslationError, \
+                "Conflicting declarations of %s within message" % \
+                placeholder.GetPresentation()
+    # update placeholder list
+    dup = 0
+    for item in self.__content:
+      if isinstance(item, Placeholder) and placeholder.EqualTo(item):
+        dup = 1
+        break
+    if not dup:
+      self.__placeholders.append(placeholder)
+
+    # update content
+    self.__content.append(placeholder)
+
+  # Strips leading and trailing whitespace, and returns a tuple
+  # containing the leading and trailing space that was removed.
+  def Strip(self):
+    leading = trailing = ''
+    if len(self.__content) > 0:
+      s0 = self.__content[0]
+      if not isinstance(s0, Placeholder):
+        s = s0.lstrip()
+        leading = s0[:-len(s)]
+        self.__content[0] = s
+
+      s0 = self.__content[-1]
+      if not isinstance(s0, Placeholder):
+        s = s0.rstrip()
+        trailing = s0[len(s):]
+        self.__content[-1] = s
+    return leading, trailing
+
+  # Return the id of this message
+  def GetId(self):
+    if self.__id is None:
+      return self.GenerateId()
+    return self.__id
+
+  # Set the id of this message
+  def SetId(self, id):
+    if id is None:
+      self.__id = None
+    else:
+      self.__id = str(id)  # Treat numerical ids as strings
+
+  # Return content of this message as a list (internal use only)
+  def GetContent(self):
+    return self.__content
+
+  # Return a human-readable version of this message
+  def GetPresentableContent(self):
+    presentable_content = ""
+    for item in self.__content:
+      if isinstance(item, Placeholder):
+        presentable_content += item.GetPresentation()
+      else:
+        presentable_content += item
+
+    return presentable_content
+
+  # Return a fragment of a message in escaped format
+  def EscapeFragment(self, fragment):
+    return fragment.replace('%', '%%')
+
+  # Return the "original" version of this message, doing %-escaping
+  # properly.  If source_msg is specified, the placeholder original
+  # information inside source_msg will be used instead.
+  def GetOriginalContent(self, source_msg = None):
+    original_content = ""
+    for item in self.__content:
+      if isinstance(item, Placeholder):
+        if source_msg:
+          ph = source_msg.GetPlaceholder(item.GetPresentation())
+          if not ph:
+            raise MessageTranslationError, \
+                  "Placeholder %s doesn't exist in message: %s" % \
+                  (item.GetPresentation(), source_msg);
+          original_content += ph.GetOriginal()
+        else:
+          original_content += item.GetOriginal()
+      else:
+        original_content += self.EscapeFragment(item)
+    return original_content
+
+  # Return the example of this message
+  def GetExampleContent(self):
+    example_content = ""
+    for item in self.__content:
+      if isinstance(item, Placeholder):
+        example_content += item.GetExample()
+      else:
+        example_content += item
+    return example_content
+
+  # Return a list of all unique placeholders in this message
+  def GetPlaceholders(self):
+    return self.__placeholders
+
+  # Return a placeholder in this message
+  def GetPlaceholder(self, presentation):
+    for item in self.__content:
+      if (isinstance(item, Placeholder) and
+          item.GetPresentation() == presentation):
+        return item
+    return None
+
+  # Return this message's description
+  def GetDescription(self):
+    return self.__description
+
+  # Add a message source
+  def AddSource(self, source):
+    self.__sources.append(source)
+
+  # Return this message's sources as a list
+  def GetSources(self):
+    return self.__sources
+
+  # Return this message's sources as a string
+  def GetSourcesAsText(self, delimiter = "; "):
+    return delimiter.join(self.__sources)
+
+  # Set the obsolete flag for a message (internal use only)
+  def SetObsolete(self):
+    self.__obsolete = 1
+
+  # Get the obsolete flag for a message (internal use only)
+  def IsObsolete(self):
+    return self.__obsolete
+
+  # Get the sequence number (0 by default)
+  def GetSequenceNumber(self):
+    return self.__sequence_number
+
+  # Set the sequence number
+  def SetSequenceNumber(self, number):
+    self.__sequence_number = number
+
+  # Increment instance counter
+  def AddInstance(self):
+    self.__num_instances += 1
+
+  # Return instance count
+  def GetNumInstances(self):
+    return self.__num_instances
+
+  def GetErrors(self, from_tc=0):
+    """
+    Returns a description of the problem if the message is not
+    syntactically valid, or None if everything is fine.
+
+    Args:
+      from_tc: indicates whether this message came from the TC. We let
+      the TC get away with some things we normally wouldn't allow for
+      historical reasons.
+    """
+    # check that placeholders are unambiguous
+    pos = 0
+    phs = {}
+    for item in self.__content:
+      if isinstance(item, Placeholder):
+        phs[pos] = item
+        pos += len(item.GetPresentation())
+      else:
+        pos += len(item)
+    presentation = self.GetPresentableContent()
+    for ph in self.GetPlaceholders():
+      for pos in FindOverlapping(presentation, ph.GetPresentation()):
+        # message contains the same text as a placeholder presentation
+        other_ph = phs.get(pos)
+        if ((not other_ph
+             and not IsSubstringInPlaceholder(pos, len(ph.GetPresentation()), phs))
+            or
+            (other_ph and len(other_ph.GetPresentation()) < len(ph.GetPresentation()))):
+          return  "message contains placeholder name '%s':\n%s" % (
+            ph.GetPresentation(), presentation)
+    return None
+
+
+  def __CopyTo(self, other):
+    """
+    Returns a copy of this BaseMessage.
+    """
+    assert isinstance(other,  self.__class__) or isinstance(self, other.__class__)
+    other.__source_encoding = self.__source_encoding
+    other.__content         = self.__content[:]
+    other.__description     = self.__description
+    other.__id              = self.__id
+    other.__num_instances   = self.__num_instances
+    other.__obsolete        = self.__obsolete
+    other.__name            = self.__name
+    other.__placeholders    = self.__placeholders[:]
+    other.__sequence_number = self.__sequence_number
+    other.__sources         = self.__sources[:]
+
+    return other
+
+  def HasText(self):
+    """Returns true iff this message has anything other than placeholders."""
+    for item in self.__content:
+      if not isinstance(item, Placeholder):
+        return True
+    return False
+
+# --------------------------------------------------------
+# The Message class represents original (English) messages
+
+class Message(BaseMessage):
+  # See BaseMessage constructor
+  def __init__(self, source_encoding, text=None, id=None,
+               description=None, meaning="", placeholders=None,
+               source=None, sequence_number=0, clone_from=None,
+               time_created=0, name=None, is_hidden = 0):
+
+    if clone_from is not None:
+      BaseMessage.__init__(self, None, clone_from=clone_from)
+      self.__meaning = clone_from.__meaning
+      self.__time_created = clone_from.__time_created
+      self.__is_hidden = clone_from.__is_hidden
+      return
+
+    BaseMessage.__init__(self, source_encoding, text, id, description,
+                         placeholders, source, sequence_number,
+                         name=name)
+    self.__meaning = meaning
+    self.__time_created = time_created
+    self.SetIsHidden(is_hidden)
+    
+  # String representation
+  def __str__(self):
+    s = 'source: %s, id: %s, content: "%s", meaning: "%s", ' \
+        'description: "%s"' % \
+        (self.GetSourcesAsText(), self.GetId(), self.GetPresentableContent(),
+         self.__meaning, self.GetDescription())
+    if self.GetName() is not None:
+      s += ', name: "%s"' % self.GetName()
+    placeholders = self.GetPlaceholders()
+    for i in range(len(placeholders)):
+      s += ", placeholder[%d]: %s" % (i, placeholders[i])
+    return s
+
+  # Strips leading and trailing whitespace, and returns a tuple
+  # containing the leading and trailing space that was removed.
+  def Strip(self):
+    leading = trailing = ''
+    content = self.GetContent()
+    if len(content) > 0:
+      s0 = content[0]
+      if not isinstance(s0, Placeholder):
+        s = s0.lstrip()
+        leading = s0[:-len(s)]
+        content[0] = s
+
+      s0 = content[-1]
+      if not isinstance(s0, Placeholder):
+        s = s0.rstrip()
+        trailing = s0[len(s):]
+        content[-1] = s
+    return leading, trailing
+
+  # Generate an id by hashing message content
+  def GenerateId(self):
+    self.SetId(GenerateMessageId(self.GetPresentableContent(),
+                                 self.__meaning))
+    return self.GetId()
+
+  def GetMeaning(self):
+    return self.__meaning
+
+  def GetTimeCreated(self):
+    return self.__time_created
+
+  # Equality operator
+  def EqualTo(self, other, strict = 1):
+    # Check id, meaning, content
+    if self.GetId() != other.GetId():
+      return 0
+    if self.__meaning != other.__meaning:
+      return 0
+    if self.GetPresentableContent() != other.GetPresentableContent():
+      return 0
+    # Check descriptions if comparison is strict
+    if (strict and
+        self.GetDescription() is not None and
+        other.GetDescription() is not None and
+        self.GetDescription() != other.GetDescription()):
+      return 0
+    # Check placeholders
+    ph1 = self.GetPlaceholders()
+    ph2 = other.GetPlaceholders()
+    if len(ph1) != len(ph2):
+      return 0
+    for i in range(len(ph1)):
+      if not ph1[i].EqualTo(ph2[i], strict):
+        return 0
+
+    return 1
+
+  def Copy(self):
+    """
+    Returns a copy of this Message.
+    """
+    assert isinstance(self, Message)
+    return Message(None, clone_from=self)
+
+  def SetIsHidden(self, is_hidden):
+    """Sets whether this message should be hidden.
+
+    Args:
+      is_hidden : 0 or 1 - if the message should be hidden, 0 otherwise
+    """
+    if is_hidden not in [0, 1]:
+      raise  MessageTranslationError, "is_hidden must be 0 or 1, got %s" 
+    self.__is_hidden = is_hidden
+
+  def IsHidden(self):
+    """Returns 1 if this message is hidden, and 0 otherwise."""
+    return self.__is_hidden
+
+# ----------------------------------------------------
+# The Translation class represents translated messages
+
+class Translation(BaseMessage):
+  # See BaseMessage constructor
+  def __init__(self, source_encoding, text=None, id=None,
+               description=None, placeholders=None, source=None,
+               sequence_number=0, clone_from=None, ignore_ph_errors=0,
+               name=None):
+    if clone_from is not None:
+      BaseMessage.__init__(self, None, clone_from=clone_from)
+      return
+
+    BaseMessage.__init__(self, source_encoding, text, id, description,
+                         placeholders, source, sequence_number,
+                         ignore_ph_errors=ignore_ph_errors, name=name)
+
+  # String representation
+  def __str__(self):
+    s = 'source: %s, id: %s, content: "%s", description: "%s"' % \
+        (self.GetSourcesAsText(), self.GetId(), self.GetPresentableContent(),
+         self.GetDescription());
+    placeholders = self.GetPlaceholders()
+    for i in range(len(placeholders)):
+      s += ", placeholder[%d]: %s" % (i, placeholders[i])
+    return s
+
+  # Equality operator
+  def EqualTo(self, other, strict=1):
+    # Check id and content
+    if self.GetId() != other.GetId():
+      return 0
+    if self.GetPresentableContent() != other.GetPresentableContent():
+      return 0
+    # Check placeholders
+    ph1 = self.GetPlaceholders()
+    ph2 = other.GetPlaceholders()
+    if len(ph1) != len(ph2):
+      return 0
+    for i in range(len(ph1)):
+      if not ph1[i].EqualTo(ph2[i], strict):
+        return 0
+
+    return 1
+
+  def Copy(self):
+    """
+    Returns a copy of this Translation.
+    """
+    return Translation(None, clone_from=self)

tools/grit/grit/format/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Module grit.format
+'''
+
+pass

tools/grit/grit/format/interface.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Base classes for item formatters and file formatters.
+'''
+
+
+import re
+
+
+class ItemFormatter(object):
+  '''Base class for a formatter that knows how to format a single item.'''
+  
+  def Format(self, item, lang='', begin_item=True, output_dir='.'):
+    '''Returns a Unicode string representing 'item' in the format known by this
+    item formatter, for the language 'lang'.  May be called once at the
+    start of the item (begin_item == True) and again at the end
+    (begin_item == False), or only at the start of the item (begin_item == True)
+    
+    Args:
+      item: anything
+      lang: 'en'
+      begin_item: True | False
+      output_dir: '.'
+    
+    Return:
+      u'hello'
+    '''
+    raise NotImplementedError()

tools/grit/grit/format/rc.py

@@ -0,0 +1,456 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Support for formatting an RC file for compilation.
+'''
+
+import os
+import types
+import re
+
+from grit import util
+from grit.format import interface
+
+
+# Matches all different types of linebreaks.
+_LINEBREAKS = re.compile('\r\n|\n|\r')
+
+'''
+This dictionary defines the langauge charset pair lookup table, which is used
+for replacing the GRIT expand variables for language info in Product Version 
+resource. The key is the language ISO country code, and the value 
+is the language and character-set pair, which is a hexadecimal string
+consisting of the concatenation of the language and character-set identifiers.
+The first 4 digit of the value is the hex value of LCID, the remaining 
+4 digits is the hex value of character-set id(code page)of the language.
+ 
+We have defined three GRIT expand_variables to be used in the version resource
+file to set the language info. Here is an example how they should be used in 
+the VS_VERSION_INFO section of the resource file to allow GRIT to localize 
+the language info correctly according to product locale.
+
+VS_VERSION_INFO VERSIONINFO
+... 
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+      BLOCK "[GRITVERLANGCHARSETHEX]"
+        BEGIN
+        ...
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", [GRITVERLANGID], [GRITVERCHARSETID]
+    END
+END
+
+'''
+
+_LANGUAGE_CHARSET_PAIR = {
+  'ar'    : '040104e8',
+  'fi'    : '040b04e4',
+  'ko'    : '041203b5',
+  'es'    : '040a04e4',
+  'bg'    : '040204e3',
+  'fr'    : '040c04e4',
+  'lv'    : '042604e9',
+  'sv'    : '041d04e4',
+  'ca'    : '040304e4',
+  'de'    : '040704e4',
+  'lt'    : '042704e9',
+  'tl'    : '0c0004b0',   # no lcid for tl(Tagalog), use default custom locale
+  'zh-CN' : '080403a8',
+  'el'    : '040804e5',
+  'no'    : '041404e4',
+  'th'    : '041e036a',
+  'zh-TW' : '040403b6',
+  'iw'    : '040d04e7',
+  'pl'    : '041504e2',
+  'tr'    : '041f04e6',
+  'hr'    : '041a04e4',
+  'hi'    : '043904b0',   # no codepage for hindi, use unicode(1200)
+  'pt-BR' : '041604e4',
+  'uk'    : '042204e3',
+  'cs'    : '040504e2',
+  'hu'    : '040e04e2',
+  'ro'    : '041804e2',
+  'ur'    : '042004b0',   # no codepage for urdu, use unicode(1200)
+  'da'    : '040604e4',
+  'is'    : '040f04e4',
+  'ru'    : '041904e3',
+  'vi'    : '042a04ea',
+  'nl'    : '041304e4',
+  'id'    : '042104e4',
+  'sr'    : '081a04e2',
+  'en-GB' : '0809040e',
+  'it'    : '041004e4',
+  'sk'    : '041b04e2',
+  'et'    : '042504e9',
+  'ja'    : '041103a4',
+  'sl'    : '042404e2',  
+  'en'    : '040904b0',
+}
+
+_LANGUAGE_DIRECTIVE_PAIR = {
+  'ar'    : 'LANG_ARABIC, SUBLANG_DEFAULT',
+  'fi'    : 'LANG_FINNISH, SUBLANG_DEFAULT',
+  'ko'    : 'LANG_KOREAN, SUBLANG_KOREAN',
+  'es'    : 'LANG_SPANISH, SUBLANG_SPANISH_MODERN',
+  'bg'    : 'LANG_BULGARIAN, SUBLANG_DEFAULT',
+  'fr'    : 'LANG_FRENCH, SUBLANG_FRENCH',
+  'lv'    : 'LANG_LATVIAN, SUBLANG_DEFAULT',
+  'sv'    : 'LANG_SWEDISH, SUBLANG_SWEDISH',
+  'ca'    : 'LANG_CATALAN, SUBLANG_DEFAULT',
+  'de'    : 'LANG_GERMAN, SUBLANG_GERMAN',
+  'lt'    : 'LANG_LITHUANIAN, SUBLANG_LITHUANIAN',
+  'tl'    : 'LANG_NEUTRAL, SUBLANG_DEFAULT',
+  'zh-CN' : 'LANG_CHINESE, SUBLANG_CHINESE_SIMPLIFIED',
+  'el'    : 'LANG_GREEK, SUBLANG_DEFAULT',
+  'no'    : 'LANG_NORWEGIAN, SUBLANG_DEFAULT',
+  'th'    : 'LANG_THAI, SUBLANG_DEFAULT',
+  'zh-TW' : 'LANG_CHINESE, SUBLANG_CHINESE_TRADITIONAL',
+  'iw'    : 'LANG_HEBREW, SUBLANG_DEFAULT',
+  'pl'    : 'LANG_POLISH, SUBLANG_DEFAULT',
+  'tr'    : 'LANG_TURKISH, SUBLANG_DEFAULT',
+  'hr'    : 'LANG_CROATIAN, SUBLANG_DEFAULT',
+  'hi'    : 'LANG_HINDI, SUBLANG_DEFAULT',
+  'pt-BR' : 'LANG_PORTUGUESE, SUBLANG_DEFAULT',
+  'uk'    : 'LANG_UKRAINIAN, SUBLANG_DEFAULT',
+  'cs'    : 'LANG_CZECH, SUBLANG_DEFAULT',
+  'hu'    : 'LANG_HUNGARIAN, SUBLANG_DEFAULT',
+  'ro'    : 'LANG_ROMANIAN, SUBLANG_DEFAULT',
+  'ur'    : 'LANG_URDU, SUBLANG_DEFAULT',
+  'da'    : 'LANG_DANISH, SUBLANG_DEFAULT',
+  'is'    : 'LANG_ICELANDIC, SUBLANG_DEFAULT',
+  'ru'    : 'LANG_RUSSIAN, SUBLANG_DEFAULT',
+  'vi'    : 'LANG_VIETNAMESE, SUBLANG_DEFAULT',
+  'nl'    : 'LANG_DUTCH, SUBLANG_DEFAULT',
+  'id'    : 'LANG_INDONESIAN, SUBLANG_DEFAULT',
+  'sr'    : 'LANG_SERBIAN, SUBLANG_SERBIAN_CYRILLIC',
+  'en-GB' : 'LANG_ENGLISH, SUBLANG_ENGLISH_UK',
+  'it'    : 'LANG_ITALIAN, SUBLANG_DEFAULT',
+  'sk'    : 'LANG_SLOVAK, SUBLANG_DEFAULT',
+  'et'    : 'LANG_ESTONIAN, SUBLANG_DEFAULT',
+  'ja'    : 'LANG_JAPANESE, SUBLANG_DEFAULT',
+  'sl'    : 'LANG_SLOVENIAN, SUBLANG_DEFAULT',  
+  'en'    : 'LANG_ENGLISH, SUBLANG_ENGLISH_US',
+}
+
+def GetLangCharsetPair(language) :  
+  if _LANGUAGE_CHARSET_PAIR.has_key(language) :
+    return _LANGUAGE_CHARSET_PAIR[language]
+  else :
+    print 'Warning:GetLangCharsetPair() found undefined language %s' %(language)
+    return ''
+
+def GetLangDirectivePair(language) :  
+  if _LANGUAGE_DIRECTIVE_PAIR.has_key(language) :
+    return _LANGUAGE_DIRECTIVE_PAIR[language]
+  else :
+    print 'Warning:GetLangDirectivePair() found undefined language %s' %(language)
+    return 'unknown language: see tools/grit/format/rc.py'
+
+def GetLangIdHex(language) :  
+  if _LANGUAGE_CHARSET_PAIR.has_key(language) :
+    langcharset = _LANGUAGE_CHARSET_PAIR[language]  
+    lang_id = '0x' + langcharset[0:4]  
+    return lang_id
+  else :
+    print 'Warning:GetLangIdHex() found undefined language %s' %(language)
+    return ''
+
+
+def GetCharsetIdDecimal(language) :
+  if _LANGUAGE_CHARSET_PAIR.has_key(language) :
+    langcharset = _LANGUAGE_CHARSET_PAIR[language]  
+    charset_decimal = int(langcharset[4:], 16)  
+    return str(charset_decimal)
+  else :
+    print 'Warning:GetCharsetIdDecimal() found undefined language %s' %(language)
+    return ''
+
+
+def GetUnifiedLangCode(language) :
+  r = re.compile('([a-z]{1,2})_([a-z]{1,2})')
+  if r.match(language) :
+    underscore = language.find('_')
+    return language[0:underscore] + '-' + language[underscore + 1:].upper()
+  else :
+    return language
+
+    
+def _MakeRelativePath(base_path, path_to_make_relative):
+  '''Returns a relative path such from the base_path to
+  the path_to_make_relative.
+  
+  In other words, os.join(base_path,
+    MakeRelativePath(base_path, path_to_make_relative))
+  is the same location as path_to_make_relative.
+      
+  Args:
+    base_path: the root path
+    path_to_make_relative: an absolute path that is on the same drive
+      as base_path
+  '''
+
+  def _GetPathAfterPrefix(prefix_path, path_with_prefix):
+    '''Gets the subpath within in prefix_path for the path_with_prefix
+    with no beginning or trailing path separators.
+      
+    Args:
+      prefix_path: the base path
+      path_with_prefix: a path that starts with prefix_path
+    '''
+    assert path_with_prefix.startswith(prefix_path)
+    path_without_prefix = path_with_prefix[len(prefix_path):]
+    normalized_path = os.path.normpath(path_without_prefix.strip(os.path.sep))
+    if normalized_path == '.':
+      normalized_path = ''
+    return normalized_path
+  
+  def _GetCommonBaseDirectory(*args):
+    '''Returns the common prefix directory for the given paths
+
+    Args:
+      The list of paths (at least one of which should be a directory)
+    '''
+    prefix = os.path.commonprefix(args)
+    # prefix is a character-by-character prefix (i.e. it does not end
+    # on a directory bound, so this code fixes that)
+
+    # if the prefix ends with the separator, then it is prefect.
+    if len(prefix) > 0 and prefix[-1] == os.path.sep:
+      return prefix
+
+    # We need to loop through all paths or else we can get
+    # tripped up by "c:\a" and "c:\abc".  The common prefix
+    # is "c:\a" which is a directory and looks good with
+    # respect to the first directory but it is clear that
+    # isn't a common directory when the second path is
+    # examined.
+    for path in args:
+      assert len(path) >= len(prefix)
+      # If the prefix the same length as the path,
+      # then the prefix must be a directory (since one
+      # of the arguements should be a directory).
+      if path == prefix:
+        continue
+      # if the character after the prefix in the path
+      # is the separator, then the prefix appears to be a
+      # valid a directory as well for the given path
+      if path[len(prefix)] == os.path.sep:
+        continue
+      # Otherwise, the prefix is not a directory, so it needs
+      # to be shortened to be one
+      index_sep = prefix.rfind(os.path.sep)
+      # The use "index_sep + 1" because it includes the final sep
+      # and it handles the case when the index_sep is -1 as well
+      prefix = prefix[:index_sep + 1]
+      # At this point we backed up to a directory bound which is
+      # common to all paths, so we can quit going through all of
+      # the paths.
+      break
+    return prefix          
+
+  prefix =  _GetCommonBaseDirectory(base_path, path_to_make_relative)
+  # If the paths had no commonality at all, then return the absolute path
+  # because it is the best that can be done.  If the path had to be relative
+  # then eventually this absolute path will be discovered (when a build breaks)
+  # and an appropriate fix can be made, but having this allows for the best 
+  # backward compatibility with the absolute path behavior in the past.
+  if len(prefix) <= 0:
+    return path_to_make_relative
+  # Build a path from the base dir to the common prefix
+  remaining_base_path = _GetPathAfterPrefix(prefix, base_path)
+
+  #  The follow handles two case: "" and "foo\\bar"
+  path_pieces = remaining_base_path.split(os.path.sep)
+  base_depth_from_prefix = len([d for d in path_pieces if len(d)])
+  base_to_prefix = (".." + os.path.sep) * base_depth_from_prefix
+  
+  # Put add in the path from the prefix to the path_to_make_relative
+  remaining_other_path = _GetPathAfterPrefix(prefix, path_to_make_relative)
+  return base_to_prefix + remaining_other_path
+
+
+class TopLevel(interface.ItemFormatter):
+  '''Writes out the required preamble for RC files.'''
+  def Format(self, item, lang='en', begin_item=True, output_dir='.'):
+    assert isinstance(lang, types.StringTypes)
+    if not begin_item:
+      return ''
+    else:
+      # Find the location of the resource header file, so that we can include
+      # it.
+      resource_header = 'resource.h'  # fall back to this
+      language_directive = ''
+      for child in item.GetRoot().children:
+        if child.name == 'outputs':
+          for output in child.children:
+            if output.attrs['type'] == 'rc_header':
+              resource_header = os.path.abspath(output.GetOutputFilename())
+              resource_header = _MakeRelativePath(output_dir, resource_header)
+            if output.attrs['lang'] != lang:
+              continue
+            if output.attrs['language_section'] == '':
+              # If no language_section is requested, no directive is added
+              # (Used when the generated rc will be included from another rc 
+              # file that will have the appropriate language directive)
+              language_directive = ''
+            elif output.attrs['language_section'] == 'neutral':
+              # If a neutral language section is requested (default), add a 
+              # neutral language directive
+              language_directive = 'LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL'
+            elif output.attrs['language_section'] == 'lang':
+              language_directive = 'LANGUAGE %s' % GetLangDirectivePair(lang)
+      resource_header = resource_header.replace('\\', '\\\\')
+      return '''// Copyright (c) Google Inc. %d
+// All rights reserved.
+// This file is automatically generated by GRIT.  Do not edit.
+
+#include "%s"
+#include <winres.h>
+#include <winresrc.h>
+
+%s
+
+
+''' % (util.GetCurrentYear(), resource_header, language_directive)
+# end Format() function
+
+
+
+class StringTable(interface.ItemFormatter):
+  '''Surrounds a collection of string messages with the required begin and
+  end blocks to declare a string table.'''
+
+  def Format(self, item, lang='en', begin_item=True, output_dir='.'):
+    assert isinstance(lang, types.StringTypes)
+    if begin_item:
+      return 'STRINGTABLE\nBEGIN\n'
+    else:
+      return 'END\n\n'
+
+
+class Message(interface.ItemFormatter):
+  '''Writes out a single message to a string table.'''
+  
+  def Format(self, item, lang='en', begin_item=True, output_dir='.'):
+    from grit.node import message
+    if not begin_item:
+      return ''
+    
+    assert isinstance(lang, types.StringTypes)
+    assert isinstance(item, message.MessageNode)
+    
+    message = item.ws_at_start + item.Translate(lang) + item.ws_at_end
+    # Escape quotation marks (RC format uses doubling-up
+    message = message.replace('"', '""')
+    # Replace linebreaks with a \n escape
+    message = _LINEBREAKS.sub(r'\\n', message)
+    
+    name_attr = item.GetTextualIds()[0]
+    
+    return '  %-15s "%s"\n' % (name_attr, message)
+
+
+class RcSection(interface.ItemFormatter):
+  '''Writes out an .rc file section.'''
+  
+  def Format(self, item, lang='en', begin_item=True, output_dir='.'):
+    if not begin_item:
+      return ''
+    
+    assert isinstance(lang, types.StringTypes)
+    from grit.node import structure
+    assert isinstance(item, structure.StructureNode)
+    
+    if item.IsExcludedFromRc():
+      return ''
+    else:
+      text = item.gatherer.Translate(
+        lang, skeleton_gatherer=item.GetSkeletonGatherer(),
+        pseudo_if_not_available=item.PseudoIsAllowed(),
+        fallback_to_english=item.ShouldFallbackToEnglish()) + '\n\n'
+
+      # Replace the language expand_variables in version rc info.
+      unified_lang_code = GetUnifiedLangCode(lang)
+      text = text.replace('[GRITVERLANGCHARSETHEX]', 
+                          GetLangCharsetPair(unified_lang_code))
+      text = text.replace('[GRITVERLANGID]', GetLangIdHex(unified_lang_code))
+      text = text.replace('[GRITVERCHARSETID]',
+                          GetCharsetIdDecimal(unified_lang_code))
+
+      return text
+
+
+class RcInclude(interface.ItemFormatter):
+  '''Writes out an item that is included in an .rc file (e.g. an ICON)'''
+  
+  def __init__(self, type, filenameWithoutPath = 0, relative_path = 0):
+    '''Indicates to the instance what the type of the resource include is,
+    e.g. 'ICON' or 'HTML'.  Case must be correct, i.e. if the type is all-caps
+    the parameter should be all-caps.
+    
+    Args:
+      type: 'ICON'
+    '''
+    self.type_ = type
+    self.filenameWithoutPath = filenameWithoutPath
+    self.relative_path_ = relative_path
+  
+  def Format(self, item, lang='en', begin_item=True, output_dir='.'):
+    if not begin_item:
+      return ''
+    
+    assert isinstance(lang, types.StringTypes)
+    from grit.node import structure
+    from grit.node import include
+    assert isinstance(item, (structure.StructureNode, include.IncludeNode))
+    assert (isinstance(item, include.IncludeNode) or
+            item.attrs['type'] in ['tr_html', 'admin_template', 'txt', 'muppet'])
+        
+    # By default, we use relative pathnames to included resources so that
+    # sharing the resulting .rc files is possible.
+    #
+    # The FileForLanguage() Function has the side effect of generating the file
+    # if needed (e.g. if it is an HTML file include).
+    filename = os.path.abspath(item.FileForLanguage(lang, output_dir))
+    if self.filenameWithoutPath:
+      filename = os.path.basename(filename)
+    elif self.relative_path_:
+      filename = _MakeRelativePath(output_dir, filename)
+    filename = filename.replace('\\', '\\\\')  # escape for the RC format
+
+    if isinstance(item, structure.StructureNode) and item.IsExcludedFromRc():
+      return ''
+    else:
+      return '%-18s %-18s "%s"\n' % (item.attrs['name'], self.type_, filename)

tools/grit/grit/format/rc_header.py

@@ -0,0 +1,207 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Item formatters for RC headers.
+'''
+
+import re
+import time
+
+from grit.format import interface
+from grit import exception
+from grit import util
+
+from grit.extern import FP
+
+
+class TopLevel(interface.ItemFormatter):
+  '''Writes the necessary preamble for a resource.h file.'''
+  
+  def Format(self, item, lang='', begin_item=True, output_dir='.'):
+    print 'TopLevel formatter called for item %s, begin %s' % (item, begin_item)
+    if not begin_item:
+      return ''
+    else:
+      header_string = '''// Copyright (c) Google Inc. %d
+// All rights reserved.
+// This file is automatically generated by GRIT. Do not edit.
+// Built on %s
+
+#pragma once
+''' % (util.GetCurrentYear(), time.asctime())
+      # Check for emit nodes under the rc_header. If any emit node
+      # is present, we assume it means the GRD file wants to override
+      # the default header, with no includes.
+      for output_node in item.GetOutputFiles():
+        if output_node.GetType() == 'rc_header':
+          for child in output_node.children:
+            if child.name == 'emit':
+              if child.attrs['emit_type'] == 'prepend':
+                return header_string 
+      # else print out the default header with include
+      return header_string + ''' 
+#include <atlres.h>
+
+'''
+
+
+class EmitAppender(interface.ItemFormatter):
+  '''Adds the content of the <emit> nodes to the RC header file.'''
+
+  def Format(self, item, lang='', begin_item=True, output_dir='.'):
+    if not begin_item:
+      return ''
+    else:
+      return '%s\n' % (item.GetCdata())
+
+class Item(interface.ItemFormatter):
+  '''Writes the #define line(s) for a single item in a resource.h file.  If
+  your node has multiple IDs that need to be defined (as is the case e.g. for
+  dialog resources) it should define a function GetTextIds(self) that returns
+  a list of textual IDs (strings).  Otherwise the formatter will use the
+  'name' attribute of the node.'''
+  
+  # All IDs allocated so far, mapped to the textual ID they represent.
+  # Used to detect and resolve collisions.
+  ids_ = {}
+
+  # All textual IDs allocated so far, mapped to the numerical ID they
+  # represent. Used when literal IDs are being defined in the 'identifiers'
+  # section of the GRD file to define other message IDs.
+  tids_ = {}
+  
+  def _VerifyId(self, id, tid, msg_if_error):
+    if id in self.ids_ and self.ids_[id] != tid:
+      raise exception.IdRangeOverlap(msg_if_error +
+        '\nUse the first_id attribute on grouping nodes (<structures>,\n'
+        '<includes>, <messages> and <ids>) to fix this problem.')
+    if id < 101:
+      print ('WARNING: Numeric resource IDs should be greater than 100 to avoid\n'
+             'conflicts with system-defined resource IDs.')
+  
+  def Format(self, item, lang='', begin_item=True, output_dir='.'):
+    if not begin_item:
+      return ''
+
+    # Resources that use the RES protocol don't need
+    # any numerical ids generated, so we skip them altogether.
+    # This is accomplished by setting the flag 'generateid' to false
+    # in the GRD file.
+    if 'generateid' in item.attrs:
+      if item.attrs['generateid'] == 'false':
+        return ''
+    
+    text_ids = item.GetTextualIds()
+
+    # We consider the "parent" of the item to be the GroupingNode containing
+    # the item, as its immediate parent may be an <if> node.
+    item_parent = item.parent
+    import grit.node.empty
+    while item_parent and not isinstance(item_parent,
+                                         grit.node.empty.GroupingNode):
+      item_parent = item_parent.parent
+    
+    lines = []
+    for tid in text_ids:
+      if util.SYSTEM_IDENTIFIERS.match(tid):
+        # Don't emit a new ID for predefined IDs
+        continue
+
+      # Some identifier nodes can provide their own id,
+      # and we use that id in the generated header in that case.
+      if hasattr(item, 'GetId') and item.GetId():
+        id = long(item.GetId())
+
+      elif ('offset' in item.attrs and item_parent and 
+            'first_id' in item_parent.attrs and item_parent.attrs['first_id'] != ''):
+         offset_text = item.attrs['offset']
+         parent_text = item_parent.attrs['first_id']
+         
+         try:
+          offset_id = long(offset_text)
+         except ValueError:
+          offset_id = self.tids_[offset_text]
+
+         try:
+          parent_id = long(parent_text)
+         except ValueError: 
+          parent_id = self.tids_[parent_text]
+      
+         id = parent_id + offset_id
+        
+      # We try to allocate IDs sequentially for blocks of items that might
+      # be related, for instance strings in a stringtable (as their IDs might be
+      # used e.g. as IDs for some radio buttons, in which case the IDs must
+      # be sequential).
+      #
+      # We do this by having the first item in a section store its computed ID
+      # (computed from a fingerprint) in its parent object.  Subsequent children
+      # of the same parent will then try to get IDs that sequentially follow
+      # the currently stored ID (on the parent) and increment it.
+      elif not item_parent or not hasattr(item_parent, '_last_id_'):
+        # First check if the starting ID is explicitly specified by the parent.
+        if (item_parent and 'first_id' in item_parent.attrs and
+            item_parent.attrs['first_id'] != ''):
+          id = long(item_parent.attrs['first_id'])
+          self._VerifyId(id, tid,
+            'Explicitly specified numeric first_id %d conflicts with one of the\n'
+            'ID ranges already used.' % id)
+        else:
+          # Automatically generate the ID based on the first clique from the
+          # first child of the first child node of our parent (i.e. when we
+          # first get to this location in the code).
+          
+          # According to
+          # http://msdn.microsoft.com/en-us/library/t2zechd4(VS.71).aspx
+          # the safe usable range for resource IDs in Windows is from decimal
+          # 101 to 0x7FFF.
+
+          id = FP.UnsignedFingerPrint(tid)
+          id = id % (0x7FFF - 101)
+          id += 101
+        
+          self._VerifyId(id, tid,
+            'Automatic (fingerprint-based) numeric ID for %s (%d) overlapped\n'
+            'with a previously allocated range.' % (tid, id))
+          
+        if item_parent:
+          item_parent._last_id_ = id
+      else:
+        assert hasattr(item_parent, '_last_id_')
+        id = item_parent._last_id_ = item_parent._last_id_ + 1
+        self._VerifyId(id, tid,
+          'Wanted to make numeric value for ID %s (%d) follow the numeric value of\n'
+          'the previous ID in the .grd file, but it was already used.' % (tid, id))
+      
+      if tid not in self.ids_.values():
+        self.ids_[id] = tid
+        self.tids_[tid] = id
+        lines.append('#define %s %d\n' % (tid, id))
+    return ''.join(lines)

tools/grit/grit/format/rc_header_unittest.py

@@ -0,0 +1,129 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for the rc_header formatter'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import StringIO
+import unittest
+
+from grit.format import rc_header
+from grit.node import message
+from grit.node import structure
+from grit.node import include
+from grit.node import misc
+from grit import grd_reader
+from grit import exception
+
+
+class RcHeaderFormatterUnittest(unittest.TestCase):
+  def setUp(self):
+    self.formatter = rc_header.Item()
+    self.formatter.ids_ = {}  # need to reset this between tests
+  
+  def FormatAll(self, grd):
+    output = []
+    for node in grd:
+      if isinstance(node, (message.MessageNode, structure.StructureNode,
+                           include.IncludeNode, misc.IdentifierNode)):
+        output.append(self.formatter.Format(node))
+    output = ''.join(output)
+    return output.replace(' ', '')
+  
+  def testFormatter(self):
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir=".">
+        <release seq="3">
+          <includes first_id="300" comment="bingo">
+            <include type="gif" name="ID_LOGO" file="images/logo.gif" />
+          </includes>
+          <messages first_id="10000">
+            <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+              Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+            </message>
+            <message name="IDS_BONGO">
+              Bongo!
+            </message>
+          </messages>
+          <structures>
+            <structure type="dialog" name="IDD_NARROW_DIALOG" file="rc_files/dialogs.rc" />
+            <structure type="version" name="VS_VERSION_INFO" file="rc_files/version.rc" />
+          </structures>
+        </release>
+      </grit>'''), '.')
+    output = self.FormatAll(grd)
+    self.failUnless(output.count('IDS_GREETING10000'))
+    self.failUnless(output.count('ID_LOGO300'))
+    
+  def testExplicitFirstIdOverlaps(self):
+    # second first_id will overlap preexisting range
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir=".">
+        <release seq="3">
+          <includes first_id="300" comment="bingo">
+            <include type="gif" name="ID_LOGO" file="images/logo.gif" />
+            <include type="gif" name="ID_LOGO2" file="images/logo2.gif" />
+          </includes>
+          <messages first_id="301">
+            <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+              Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+            </message>
+            <message name="IDS_SMURFGEBURF">Frubegfrums</message>
+          </messages>
+        </release>
+      </grit>'''), '.')
+    self.assertRaises(exception.IdRangeOverlap, self.FormatAll, grd)
+  
+  def testImplicitOverlapsPreexisting(self):
+    # second message in <messages> will overlap preexisting range
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en" current_release="3" base_dir=".">
+        <release seq="3">
+          <includes first_id="301" comment="bingo">
+            <include type="gif" name="ID_LOGO" file="images/logo.gif" />
+            <include type="gif" name="ID_LOGO2" file="images/logo2.gif" />
+          </includes>
+          <messages first_id="300">
+            <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+              Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+            </message>
+            <message name="IDS_SMURFGEBURF">Frubegfrums</message>
+          </messages>
+        </release>
+      </grit>'''), '.')
+    self.assertRaises(exception.IdRangeOverlap, self.FormatAll, grd)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/format/rc_unittest.py

@@ -0,0 +1,287 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.format.rc'''
+
+import os
+import re
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import tempfile
+import unittest
+import StringIO
+
+from grit.format import rc
+from grit import grd_reader
+from grit import util
+from grit.tool import build
+
+class DummyOutput(object):
+  def __init__(self, type, language):
+    self.type = type
+    self.language = language
+  def GetType(self):
+    return self.type
+  def GetLanguage(self):
+    return self.language
+  def GetOutputFilename(self):
+    return 'hello.gif'
+
+class FormatRcUnittest(unittest.TestCase):
+  def testMessages(self):
+    root = grd_reader.Parse(StringIO.StringIO('''
+      <messages>
+          <message name="IDS_BTN_GO" desc="Button text" meaning="verb">Go!</message>
+          <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+            Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+          </message>
+          <message name="BONGO" desc="Flippo nippo">
+            Howdie "Mr. Elephant", how are you doing?   \'\'\'
+          </message>
+          <message name="IDS_WITH_LINEBREAKS">
+Good day sir,
+I am a bee
+Sting sting
+          </message>
+      </messages>
+      '''), flexible_root = True)
+    util.FixRootForUnittest(root)
+    
+    buf = StringIO.StringIO()
+    build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf)
+    output = buf.getvalue()
+    self.failUnless(output.strip() == u'''
+STRINGTABLE
+BEGIN
+  IDS_BTN_GO      "Go!"
+  IDS_GREETING    "Hello %s, how are you doing today?"
+  BONGO           "Howdie ""Mr. Elephant"", how are you doing?   "
+  IDS_WITH_LINEBREAKS "Good day sir,\\nI am a bee\\nSting sting"
+END'''.strip())
+
+
+  def testRcSection(self):
+    root = grd_reader.Parse(StringIO.StringIO('''
+      <structures>
+          <structure type="menu" name="IDC_KLONKMENU" file="grit\\test\data\klonk.rc" encoding="utf-16" />
+          <structure type="dialog" name="IDD_ABOUTBOX" file="grit\\test\data\klonk.rc" encoding="utf-16" />
+          <structure type="version" name="VS_VERSION_INFO" file="grit\\test\data\klonk.rc" encoding="utf-16" />
+      </structures>'''), flexible_root = True)
+    util.FixRootForUnittest(root)
+    root.RunGatherers(recursive = True)
+    
+    buf = StringIO.StringIO()
+    build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf)
+    output = buf.getvalue()
+    self.failUnless(output.strip() == u'''
+IDC_KLONKMENU MENU 
+BEGIN
+    POPUP "&File"
+    BEGIN
+        MENUITEM "E&xit",                       IDM_EXIT
+        MENUITEM "This be ""Klonk"" me like",   ID_FILE_THISBE
+        POPUP "gonk"
+        BEGIN
+            MENUITEM "Klonk && is ""good""",           ID_GONK_KLONKIS
+        END
+    END
+    POPUP "&Help"
+    BEGIN
+        MENUITEM "&About ...",                  IDM_ABOUT
+    END
+END
+
+IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75
+STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU
+CAPTION "About"
+FONT 8, "System", 0, 0, 0x0
+BEGIN
+    ICON            IDI_KLONK,IDC_MYICON,14,9,20,20
+    LTEXT           "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8,
+                    SS_NOPREFIX
+    LTEXT           "Copyright (C) 2005",IDC_STATIC,49,20,119,8
+    DEFPUSHBUTTON   "OK",IDOK,195,6,30,11,WS_GROUP
+    CONTROL         "Jack ""Black"" Daniels",IDC_RADIO1,"Button",
+                    BS_AUTORADIOBUTTON,46,51,84,10
+END
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION 1,0,0,1
+ PRODUCTVERSION 1,0,0,1
+ FILEFLAGSMASK 0x17L
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x1L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "FileDescription", "klonk Application"
+            VALUE "FileVersion", "1, 0, 0, 1"
+            VALUE "InternalName", "klonk"
+            VALUE "LegalCopyright", "Copyright (C) 2005"
+            VALUE "OriginalFilename", "klonk.exe"
+            VALUE "ProductName", " klonk Application"
+            VALUE "ProductVersion", "1, 0, 0, 1"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END'''.strip())
+
+
+  def testRcIncludeStructure(self):
+    root = grd_reader.Parse(StringIO.StringIO('''
+      <structures>
+        <structure type="tr_html" name="IDR_HTML" file="bingo.html"/>
+        <structure type="tr_html" name="IDR_HTML2" file="bingo2.html"/>
+      </structures>'''), flexible_root = True)
+    util.FixRootForUnittest(root, '/temp')
+    # We do not run gatherers as it is not needed and wouldn't find the file
+    
+    buf = StringIO.StringIO()
+    build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf)
+    output = buf.getvalue()
+    expected = (u'IDR_HTML           HTML               "%s"\n'
+                u'IDR_HTML2          HTML               "%s"'
+                % (util.normpath('/temp/bingo.html').replace('\\', '\\\\'),
+                   util.normpath('/temp/bingo2.html').replace('\\', '\\\\')))
+    # hackety hack to work on win32&lin
+    output = re.sub('"[c-zC-Z]:', '"', output) 
+    self.failUnless(output.strip() == expected)
+
+  def testRcIncludeFile(self):
+    root = grd_reader.Parse(StringIO.StringIO('''
+      <includes>
+        <include type="TXT" name="TEXT_ONE" file="bingo.txt"/>
+        <include type="TXT" name="TEXT_TWO" file="bingo2.txt"  filenameonly="true" />
+      </includes>'''), flexible_root = True)
+    util.FixRootForUnittest(root, '/temp')
+    
+    buf = StringIO.StringIO()
+    build.RcBuilder.ProcessNode(root, DummyOutput('rc_all', 'en'), buf)
+    output = buf.getvalue()
+    expected = (u'TEXT_ONE           TXT                "%s"\n'
+                u'TEXT_TWO           TXT                "%s"'
+                % (util.normpath('/temp/bingo.txt').replace('\\', '\\\\'),
+                   'bingo2.txt'))
+    # hackety hack to work on win32&lin
+    output = re.sub('"[c-zC-Z]:', '"', output)
+    self.failUnless(output.strip() == expected)
+
+
+  def testStructureNodeOutputfile(self):
+    input_file = util.PathFromRoot('grit/test/data/simple.html')
+    root = grd_reader.Parse(StringIO.StringIO(
+      '<structure type="tr_html" name="IDR_HTML" file="%s" />' %input_file),
+      flexible_root = True)
+    util.FixRootForUnittest(root, '.')
+    # We must run the gatherers since we'll be wanting the translation of the
+    # file.  The file exists in the location pointed to.
+    root.RunGatherers(recursive=True)
+    
+    output_dir = tempfile.gettempdir()
+    en_file = root.FileForLanguage('en', output_dir)
+    self.failUnless(en_file == input_file)
+    fr_file = root.FileForLanguage('fr', output_dir)
+    self.failUnless(fr_file == os.path.join(output_dir, 'fr_simple.html'))
+    
+    fo = file(fr_file)
+    contents = fo.read()
+    fo.close()
+    
+    self.failUnless(contents.find('<p>') != -1)  # should contain the markup
+    self.failUnless(contents.find('Hello!') == -1)  # should be translated
+  
+  
+  def testFallbackToEnglish(self):
+    root = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <release seq="1" allow_pseudo="False">
+          <structures fallback_to_english="True">
+            <structure type="dialog" name="IDD_ABOUTBOX" file="grit\\test\data\klonk.rc" encoding="utf-16" />
+          </structures>
+        </release>
+      </grit>'''), util.PathFromRoot('.'))
+    util.FixRootForUnittest(root)
+    root.RunGatherers(recursive = True)
+    
+    node = root.GetNodeById("IDD_ABOUTBOX")
+    formatter = node.ItemFormatter('rc_all')
+    output = formatter.Format(node, 'bingobongo')
+    self.failUnless(output.strip() == '''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75
+STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU
+CAPTION "About"
+FONT 8, "System", 0, 0, 0x0
+BEGIN
+    ICON            IDI_KLONK,IDC_MYICON,14,9,20,20
+    LTEXT           "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8,
+                    SS_NOPREFIX
+    LTEXT           "Copyright (C) 2005",IDC_STATIC,49,20,119,8
+    DEFPUSHBUTTON   "OK",IDOK,195,6,30,11,WS_GROUP
+    CONTROL         "Jack ""Black"" Daniels",IDC_RADIO1,"Button",
+                    BS_AUTORADIOBUTTON,46,51,84,10
+END''')
+
+
+  def testRelativePath(self):
+    ''' Verify that _MakeRelativePath works in some tricky cases.'''
+    def TestRelativePathCombinations(base_path, other_path, expected_result):
+      ''' Verify that the relative path function works for
+      the given paths regardless of whether or not they end with
+      a trailing slash.'''
+      for path1 in [base_path, base_path + os.path.sep]:
+        for path2 in [other_path, other_path + os.path.sep]:
+          result = rc._MakeRelativePath(path1, path2)
+          self.failUnless(result == expected_result)
+
+    # set-up variables
+    root_dir = 'c:%sa' % os.path.sep
+    result1 = '..%sabc' % os.path.sep
+    path1 = root_dir + 'bc'
+    result2 = 'bc'
+    path2 = '%s%s%s' % (root_dir, os.path.sep, result2)
+    # run the tests
+    TestRelativePathCombinations(root_dir, path1, result1)
+    TestRelativePathCombinations(root_dir, path2, result2)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/gather/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Module grit.gather
+'''
+
+pass

tools/grit/grit/gather/admin_template.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Gatherer for administrative template files.
+'''
+
+import re
+import types
+
+from grit.gather import regexp
+from grit import exception
+from grit import tclib
+from grit import util
+
+
+class MalformedAdminTemplateException(exception.Base):
+  '''This file doesn't look like a .adm file to me.'''
+  def __init__(self, msg=''):
+    exception.Base.__init__(self, msg)
+
+
+class AdmGatherer(regexp.RegexpGatherer):
+  '''Gatherer for the translateable portions of an admin template.
+  
+  This gatherer currently makes the following assumptions:
+  - there is only one [strings] section and it is always the last section
+    of the file
+  - translateable strings do not need to be escaped.
+  '''
+
+  # Finds the strings section as the group named 'strings'
+  _STRINGS_SECTION = re.compile('(?P<first_part>.+^\[strings\])(?P<strings>.+)\Z',
+                                re.MULTILINE | re.DOTALL)
+  
+  # Finds the translateable sections from within the [strings] section.
+  _TRANSLATEABLES = re.compile('^\s*[A-Za-z0-9_]+\s*=\s*"(?P<text>.+)"\s*$',
+                               re.MULTILINE)
+  
+  def __init__(self, text):
+    regexp.RegexpGatherer.__init__(self, text)
+  
+  def Escape(self, text):
+    return text.replace('\n', '\\n')
+  
+  def UnEscape(self, text):
+    return text.replace('\\n', '\n')
+
+  def Parse(self):
+    if self.have_parsed_:
+      return
+    m = self._STRINGS_SECTION.match(self.text_)
+    if not m:
+      raise MalformedAdminTemplateException()
+    # Add the first part, which is all nontranslateable, to the skeleton
+    self._AddNontranslateableChunk(m.group('first_part'))
+    # Then parse the rest using the _TRANSLATEABLES regexp.
+    self._RegExpParse(self._TRANSLATEABLES, m.group('strings'))
+  
+  # static method
+  def FromFile(adm_file, ext_key=None, encoding='cp1252'):
+    '''Loads the contents of 'adm_file' in encoding 'encoding' and creates
+    an AdmGatherer instance that gathers from those contents.
+    
+    The 'ext_key' parameter is ignored.
+    
+    Args:
+      adm_file: file('bingo.rc') | 'filename.rc'
+      encoding: 'utf-8'
+    
+    Return:
+      AdmGatherer(contents_of_file)
+    '''
+    if isinstance(adm_file, types.StringTypes):
+      adm_file = util.WrapInputStream(file(adm_file, 'r'), encoding)
+    return AdmGatherer(adm_file.read())
+  FromFile = staticmethod(FromFile)

tools/grit/grit/gather/admin_template_unittest.py

@@ -0,0 +1,141 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for the admin template gatherer.'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import StringIO
+import tempfile
+import unittest
+
+from grit.gather import admin_template
+from grit import util
+from grit import grd_reader
+from grit import grit_runner
+from grit.tool import build
+
+
+class AdmGathererUnittest(unittest.TestCase):
+  def testParsingAndTranslating(self):
+    pseudofile = StringIO.StringIO(
+      'bingo bongo\n'
+      'ding dong\n'
+      '[strings] \n'
+      'whatcha="bingo bongo"\n'
+      'gotcha = "bingolabongola "the wise" fingulafongula" \n')
+    gatherer = admin_template.AdmGatherer.FromFile(pseudofile)
+    gatherer.Parse()
+    self.failUnless(len(gatherer.GetCliques()) == 2)
+    self.failUnless(gatherer.GetCliques()[1].GetMessage().GetRealContent() ==
+                    'bingolabongola "the wise" fingulafongula')
+    
+    translation = gatherer.Translate('en')
+    self.failUnless(translation == gatherer.GetText().strip())
+  
+  def testErrorHandling(self):
+    pseudofile = StringIO.StringIO(
+      'bingo bongo\n'
+      'ding dong\n'
+      'whatcha="bingo bongo"\n'
+      'gotcha = "bingolabongola "the wise" fingulafongula" \n')
+    gatherer = admin_template.AdmGatherer.FromFile(pseudofile)
+    self.assertRaises(admin_template.MalformedAdminTemplateException,
+                      gatherer.Parse)
+  
+  _TRANSLATABLES_FROM_FILE = (
+    'Google', 'Google Desktop Search', 'Preferences',
+    'Controls Google Deskop Search preferences',
+    'Indexing and Capture Control',
+    'Controls what files, web pages, and other content will be indexed by Google Desktop Search.',
+    'Prevent indexing of e-mail',
+    # there are lots more but we don't check any further
+  )
+  
+  def VerifyCliquesFromAdmFile(self, cliques):
+    self.failUnless(len(cliques) > 20)
+    for ix in range(len(self._TRANSLATABLES_FROM_FILE)):
+      text = cliques[ix].GetMessage().GetRealContent()
+      self.failUnless(text == self._TRANSLATABLES_FROM_FILE[ix])
+  
+  def testFromFile(self):
+    fname = util.PathFromRoot('grit/test/data/GoogleDesktopSearch.adm')
+    gatherer = admin_template.AdmGatherer.FromFile(fname)
+    gatherer.Parse()
+    cliques = gatherer.GetCliques()
+    self.VerifyCliquesFromAdmFile(cliques)
+  
+  def MakeGrd(self):
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3">
+        <release seq="3">
+          <structures>
+            <structure type="admin_template" name="IDAT_GOOGLE_DESKTOP_SEARCH"
+              file="GoogleDesktopSearch.adm" exclude_from_rc="true" />
+            <structure type="txt" name="BINGOBONGO"
+              file="README.txt" exclude_from_rc="true" />
+          </structures>
+        </release>
+        <outputs>
+          <output filename="de_res.rc" type="rc_all" lang="de" />
+        </outputs>
+      </grit>'''), util.PathFromRoot('grit/test/data'))
+    grd.RunGatherers(recursive=True)
+    return grd
+
+  def testInGrd(self):
+    grd = self.MakeGrd()
+    cliques = grd.children[0].children[0].children[0].GetCliques()
+    self.VerifyCliquesFromAdmFile(cliques)
+  
+  def testFileIsOutput(self):
+    grd = self.MakeGrd()
+    dirname = tempfile.mkdtemp()
+    try:
+      tool = build.RcBuilder()
+      tool.o = grit_runner.Options()
+      tool.output_directory = dirname
+      tool.res = grd
+      tool.Process()
+      
+      self.failUnless(os.path.isfile(
+        os.path.join(dirname, 'de_GoogleDesktopSearch.adm')))
+      self.failUnless(os.path.isfile(
+        os.path.join(dirname, 'de_README.txt')))
+    finally:
+      for f in os.listdir(dirname):
+        os.unlink(os.path.join(dirname, f))
+      os.rmdir(dirname)
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/gather/interface.py

@@ -0,0 +1,132 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Interface for all gatherers.
+'''
+
+
+from grit import clique
+
+
+class GathererBase(object):
+  '''Interface for all gatherer implementations.  Subclasses must implement
+  all methods that raise NotImplemented.'''
+
+  def __init__(self):
+    # A default uberclique that is local to this object.  Users can override
+    # this with the uberclique they are using.
+    self.uberclique = clique.UberClique()
+    # Indicates whether this gatherer is a skeleton gatherer, in which case
+    # we should not do some types of processing on the translateable bits.
+    self.is_skeleton = False
+
+  def SetUberClique(self, uberclique):
+    '''Overrides the default uberclique so that cliques created by this object
+    become part of the uberclique supplied by the user.
+    '''
+    self.uberclique = uberclique
+  
+  def SetSkeleton(self, is_skeleton):
+    self.is_skeleton = is_skeleton
+  
+  def IsSkeleton(self):
+    return self.is_skeleton
+  
+  def Parse(self):
+    '''Parses the contents of what is being gathered.'''
+    raise NotImplementedError()
+  
+  def GetText(self):
+    '''Returns the text of what is being gathered.'''
+    raise NotImplementedError()
+  
+  def GetTextualIds(self):
+    '''Returns the mnemonic IDs that need to be defined for the resource
+    being gathered to compile correctly.'''
+    return []
+    
+  def GetCliques(self):
+    '''Returns the MessageClique objects for all translateable portions.'''
+    return []
+  
+  def Translate(self, lang, pseudo_if_not_available=True,
+                skeleton_gatherer=None, fallback_to_english=False):
+    '''Returns the resource being gathered, with translateable portions filled
+    with the translation for language 'lang'.
+    
+    If pseudo_if_not_available is true, a pseudotranslation will be used for any
+    message that doesn't have a real translation available.
+    
+    If no translation is available and pseudo_if_not_available is false,
+    fallback_to_english controls the behavior.  If it is false, throw an error.
+    If it is true, use the English version of the message as its own
+    "translation".
+    
+    If skeleton_gatherer is specified, the translation will use the nontranslateable
+    parts from the gatherer 'skeleton_gatherer', which must be of the same type
+    as 'self'.
+    
+    If fallback_to_english 
+    
+    Args:
+      lang: 'en'
+      pseudo_if_not_available: True | False
+      skeleton_gatherer: other_gatherer
+      fallback_to_english: True | False
+    
+    Return:
+      e.g. 'ID_THIS_SECTION TYPE\n...BEGIN\n  "Translated message"\n......\nEND'
+    
+    Raises:
+      grit.exception.NotReady() if used before Parse() has been successfully
+      called.
+      grit.exception.NoSuchTranslation() if 'pseudo_if_not_available' and
+      fallback_to_english are both false and there is no translation for the
+      requested language.
+    '''
+    raise NotImplementedError()
+  
+  def FromFile(rc_file, extkey=None, encoding = 'cp1252'):
+    '''Loads the resource from the file 'rc_file'.  Optionally an external key
+    (which gets passed to the gatherer's constructor) can be specified.
+    
+    If 'rc_file' is a filename, it will be opened for reading using 'encoding'.
+    Otherwise the 'encoding' parameter is ignored.
+    
+    Args:
+      rc_file: file('') | 'filename.rc'
+      extkey: e.g. 'ID_MY_DIALOG'
+      encoding: 'utf-8'
+    
+    Return:
+      grit.gather.interface.GathererBase subclass
+    '''
+    raise NotImplementedError()
+  FromFile = staticmethod(FromFile)

tools/grit/grit/gather/muppet_strings.py

@@ -0,0 +1,165 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Support for "strings.xml" format used by Muppet plug-ins in Google Desktop.'''
+
+import StringIO
+import types
+import re
+import xml.sax
+import xml.sax.handler
+import xml.sax.saxutils
+
+from grit.gather import regexp
+from grit import util
+from grit import tclib
+
+# Placeholders can be defined in strings.xml files by putting the name of the
+# placeholder between [![ and ]!] e.g. <MSG>Hello [![USER]!] how are you<MSG>
+PLACEHOLDER_RE = re.compile('(\[!\[|\]!\])')
+
+
+class MuppetStringsContentHandler(xml.sax.handler.ContentHandler):
+  '''A very dumb parser for splitting the strings.xml file into translateable
+  and nontranslateable chunks.'''
+  
+  def __init__(self, parent):
+    self.curr_elem = ''
+    self.curr_text = ''
+    self.parent = parent
+    self.description = ''
+    self.meaning = ''
+    self.translateable = True
+  
+  def startElement(self, name, attrs):
+    if (name != 'strings'):
+      self.curr_elem = name
+      
+      attr_names = attrs.getQNames()
+      if 'desc' in attr_names:
+        self.description = attrs.getValueByQName('desc')
+      if 'meaning' in attr_names:
+        self.meaning = attrs.getValueByQName('meaning')
+      if 'translateable' in attr_names:
+        value = attrs.getValueByQName('translateable')
+        if value.lower() not in ['true', 'yes']:
+          self.translateable = False
+      
+      att_text = []
+      for attr_name in attr_names:
+        att_text.append(' ')
+        att_text.append(attr_name)
+        att_text.append('=')
+        att_text.append(
+          xml.sax.saxutils.quoteattr(attrs.getValueByQName(attr_name)))
+      
+      self.parent._AddNontranslateableChunk("<%s%s>" %
+                                            (name, ''.join(att_text)))
+  
+  def characters(self, content):
+    if self.curr_elem != '':
+      self.curr_text += content
+    
+  def endElement(self, name):
+    if name != 'strings':
+      self.parent.AddMessage(self.curr_text, self.description,
+                             self.meaning, self.translateable)
+      self.parent._AddNontranslateableChunk("</%s>\n" % name)
+      self.curr_elem = ''
+      self.curr_text = ''
+      self.description = ''
+      self.meaning = ''
+      self.translateable = True
+  
+  def ignorableWhitespace(self, whitespace):
+    pass
+
+class MuppetStrings(regexp.RegexpGatherer):
+  '''Supports the strings.xml format used by Muppet gadgets.'''
+  
+  def __init__(self, text):
+    if util.IsExtraVerbose():
+      print text
+    regexp.RegexpGatherer.__init__(self, text)
+
+  def AddMessage(self, msgtext, description, meaning, translateable):
+    if msgtext == '':
+      return
+    
+    msg = tclib.Message(description=description, meaning=meaning)
+    
+    unescaped_text = self.UnEscape(msgtext)
+    parts = PLACEHOLDER_RE.split(unescaped_text)
+    in_placeholder = False
+    for part in parts:
+      if part == '':
+        continue
+      elif part == '[![':
+        in_placeholder = True
+      elif part == ']!]':
+        in_placeholder = False
+      else:
+        if in_placeholder:
+          msg.AppendPlaceholder(tclib.Placeholder(part, '[![%s]!]' % part,
+                                                  '(placeholder)'))
+        else:
+          msg.AppendText(part)
+    
+    self.skeleton_.append(
+      self.uberclique.MakeClique(msg, translateable=translateable))
+    
+    # if statement needed because this is supposed to be idempotent (so never
+    # set back to false)
+    if translateable:
+      self.translatable_chunk_ = True
+  
+  # Although we use the RegexpGatherer base class, we do not use the
+  # _RegExpParse method of that class to implement Parse().  Instead, we
+  # parse using a SAX parser.
+  def Parse(self):
+    if (self.have_parsed_):
+      return
+    self._AddNontranslateableChunk(u'<strings>\n')
+    stream = StringIO.StringIO(self.text_)
+    handler = MuppetStringsContentHandler(self)
+    xml.sax.parse(stream, handler)
+    self._AddNontranslateableChunk(u'</strings>\n')
+  
+  def Escape(self, text):
+    return util.EncodeCdata(text)
+  
+  def FromFile(filename_or_stream, extkey=None, encoding='cp1252'):
+    if isinstance(filename_or_stream, types.StringTypes):
+      if util.IsVerbose():
+        print "MuppetStrings reading file %s, encoding %s" % (
+          filename_or_stream, encoding)
+      filename_or_stream = util.WrapInputStream(file(filename_or_stream, 'r'), encoding)
+    return MuppetStrings(filename_or_stream.read())
+  FromFile = staticmethod(FromFile)

tools/grit/grit/gather/muppet_strings_unittest.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.gather.muppet_strings'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import unittest
+
+from grit.gather import muppet_strings
+
+class MuppetStringsUnittest(unittest.TestCase):
+  def testParsing(self):
+    original = '''<strings><BLA desc="Says hello">hello!</BLA><BINGO>YEEEESSS!!!</BINGO></strings>'''
+    gatherer = muppet_strings.MuppetStrings(original)
+    gatherer.Parse()
+    self.failUnless(len(gatherer.GetCliques()) == 2)
+    self.failUnless(gatherer.Translate('en').replace('\n', '') == original)
+  
+  def testEscapingAndLinebreaks(self):
+    original = ('''\
+<strings>
+<LINEBREAK desc="Howdie">Hello
+there
+how
+are
+you?</LINEBREAK>   <ESCAPED meaning="bingo">4 &lt; 6</ESCAPED>
+</strings>''')
+    gatherer = muppet_strings.MuppetStrings(original)
+    gatherer.Parse()
+    self.failUnless(gatherer.GetCliques()[0].translateable)
+    self.failUnless(len(gatherer.GetCliques()) == 2)
+    self.failUnless(gatherer.GetCliques()[0].GetMessage().GetRealContent() ==
+                    'Hello\nthere\nhow\nare\nyou?')
+    self.failUnless(gatherer.GetCliques()[0].GetMessage().GetDescription() == 'Howdie')
+    self.failUnless(gatherer.GetCliques()[1].GetMessage().GetRealContent() ==
+                    '4 < 6')
+    self.failUnless(gatherer.GetCliques()[1].GetMessage().GetMeaning() == 'bingo')
+
+  def testPlaceholders(self):
+    original = "<strings><MESSAGE translateable='True'>Hello [![USER]!] how are you? [![HOUR]!]:[![MINUTE]!]</MESSAGE></strings>"
+    gatherer = muppet_strings.MuppetStrings(original)
+    gatherer.Parse()
+    self.failUnless(gatherer.GetCliques()[0].translateable)
+    msg = gatherer.GetCliques()[0].GetMessage()
+    self.failUnless(len(msg.GetPlaceholders()) == 3)
+    ph = msg.GetPlaceholders()[0]
+    self.failUnless(ph.GetOriginal() == '[![USER]!]')
+    self.failUnless(ph.GetPresentation() == 'USER')
+  
+  def testTranslateable(self):
+    original = "<strings><BINGO translateable='false'>Yo yo hi there</BINGO></strings>"
+    gatherer = muppet_strings.MuppetStrings(original)
+    gatherer.Parse()
+    msg = gatherer.GetCliques()[0].GetMessage()
+    self.failUnless(msg.GetRealContent() == "Yo yo hi there")
+    self.failUnless(not gatherer.GetCliques()[0].translateable)
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/gather/rc.py

@@ -0,0 +1,427 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Support for gathering resources from RC files.
+'''
+
+
+import re
+import types
+
+from grit import clique
+from grit import exception
+from grit import util
+from grit import tclib
+
+from grit.gather import regexp
+
+
+# Find portions that need unescaping in resource strings.  We need to be
+# careful that a \\n is matched _first_ as a \\ rather than matching as
+# a \ followed by a \n.
+# TODO(joi) Handle ampersands if we decide to change them into <ph>
+# TODO(joi) May need to handle other control characters than \n
+_NEED_UNESCAPE = re.compile(r'""|\\\\|\\n|\\t')
+
+# Find portions that need escaping to encode string as a resource string.
+_NEED_ESCAPE = re.compile(r'"|\n|\t|\\|\&nbsp\;')
+
+# How to escape certain characters
+_ESCAPE_CHARS = {
+  '"' : '""',
+  '\n' : '\\n',
+  '\t' : '\\t',
+  '\\' : '\\\\',
+  '&nbsp;' : ' '
+}
+
+# How to unescape certain strings
+_UNESCAPE_CHARS = dict([[value, key] for key, value in _ESCAPE_CHARS.items()])
+
+
+
+class Section(regexp.RegexpGatherer):
+  '''A section from a resource file.'''
+  
+  def __init__(self, section_text):
+    '''Creates a new object.
+    
+    Args:
+      section_text: 'ID_SECTION_ID SECTIONTYPE\n.....\nBEGIN\n.....\nEND'
+    '''
+    regexp.RegexpGatherer.__init__(self, section_text)
+  
+  # static method
+  def Escape(text):
+    '''Returns a version of 'text' with characters escaped that need to be
+    for inclusion in a resource section.'''
+    def Replace(match):
+      return _ESCAPE_CHARS[match.group()]
+    return _NEED_ESCAPE.sub(Replace, text)
+  Escape = staticmethod(Escape)
+  
+  # static method  
+  def UnEscape(text):
+    '''Returns a version of 'text' with escaped characters unescaped.'''
+    def Replace(match):
+      return _UNESCAPE_CHARS[match.group()]
+    return _NEED_UNESCAPE.sub(Replace, text)
+  UnEscape = staticmethod(UnEscape)
+  
+  def _RegExpParse(self, rexp, text_to_parse):
+    '''Overrides _RegExpParse to add shortcut group handling.  Otherwise
+    the same.
+    '''
+    regexp.RegexpGatherer._RegExpParse(self, rexp, text_to_parse)
+    
+    if not self.IsSkeleton() and len(self.GetTextualIds()) > 0:
+      group_name = self.GetTextualIds()[0]
+      for c in self.GetCliques():
+        c.AddToShortcutGroup(group_name)
+  
+  # Static method
+  def FromFileImpl(rc_file, extkey, encoding, type):
+    '''Implementation of FromFile.  Need to keep separate so we can have
+    a FromFile in this class that has its type set to Section by default.
+    '''
+    if isinstance(rc_file, types.StringTypes):
+      rc_file = util.WrapInputStream(file(rc_file, 'r'), encoding)
+    
+    out = ''
+    begin_count = 0
+    for line in rc_file.readlines():
+      if len(out) > 0 or (line.strip().startswith(extkey) and
+                          line.strip().split()[0] == extkey):
+        out += line
+      
+      # we stop once we reach the END for the outermost block.
+      begin_count_was = begin_count
+      if len(out) > 0 and line.strip() == 'BEGIN':
+        begin_count += 1
+      elif len(out) > 0 and line.strip() == 'END':
+        begin_count -= 1
+      if begin_count_was == 1 and begin_count == 0:
+        break
+    
+    if len(out) == 0:
+      raise exception.SectionNotFound('%s in file %s' % (extkey, rc_file))
+
+    return type(out)
+  FromFileImpl = staticmethod(FromFileImpl)
+  
+  # static method
+  def FromFile(rc_file, extkey, encoding='cp1252'):
+    '''Retrieves the section of 'rc_file' that has the key 'extkey'.  This is
+    matched against the start of a line, and that line and the rest of that
+    section in the RC file is returned.
+    
+    If 'rc_file' is a filename, it will be opened for reading using 'encoding'.
+    Otherwise the 'encoding' parameter is ignored.
+    
+    This method instantiates an object of type 'type' with the text from the
+    file.
+    
+    Args:
+      rc_file: file('') | 'filename.rc'
+      extkey: 'ID_MY_DIALOG'
+      encoding: 'utf-8'
+      type: class to instantiate with text of section
+    
+    Return:
+      type(text_of_section)
+    '''
+    return Section.FromFileImpl(rc_file, extkey, encoding, Section)
+  FromFile = staticmethod(FromFile)
+
+
+class Dialog(Section):
+  '''A resource section that contains a dialog resource.'''
+
+  # A typical dialog resource section looks like this:
+  #
+  # IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75
+  # STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU
+  # CAPTION "About"
+  # FONT 8, "System", 0, 0, 0x0
+  # BEGIN
+  #     ICON            IDI_KLONK,IDC_MYICON,14,9,20,20
+  #     LTEXT           "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8,
+  #                     SS_NOPREFIX
+  #     LTEXT           "Copyright (C) 2005",IDC_STATIC,49,20,119,8
+  #     DEFPUSHBUTTON   "OK",IDOK,195,6,30,11,WS_GROUP
+  #     CONTROL         "Jack ""Black"" Daniels",IDC_RADIO1,"Button",
+  #                     BS_AUTORADIOBUTTON,46,51,84,10
+  # END
+  
+  # We are using a sorted set of keys, and we assume that the
+  # group name used for descriptions (type) will come after the "text"
+  # group in alphabetical order. We also assume that there cannot be
+  # more than one description per regular expression match.
+  # If that's not the case some descriptions will be clobbered.
+  dialog_re_ = re.compile('''
+    # The dialog's ID in the first line
+    (?P<id1>[A-Z0-9_]+)\s+DIALOG(EX)?
+    |
+    # The caption of the dialog
+    (?P<type1>CAPTION)\s+"(?P<text1>.*?([^"]|""))"\s
+    |
+    # Lines for controls that have text and an ID
+    \s+(?P<type2>[A-Z]+)\s+"(?P<text2>.*?([^"]|"")?)"\s*,\s*(?P<id2>[A-Z0-9_]+)\s*,
+    |
+    # Lines for controls that have text only
+    \s+(?P<type3>[A-Z]+)\s+"(?P<text3>.*?([^"]|"")?)"\s*,
+    |
+    # Lines for controls that reference other resources
+    \s+[A-Z]+\s+[A-Z0-9_]+\s*,\s*(?P<id3>[A-Z0-9_]*[A-Z][A-Z0-9_]*)
+    |
+    # This matches "NOT SOME_STYLE" so that it gets consumed and doesn't get
+    # matched by the next option (controls that have only an ID and then just
+    # numbers)
+    \s+NOT\s+[A-Z][A-Z0-9_]+
+    |
+    # Lines for controls that have only an ID and then just numbers
+    \s+[A-Z]+\s+(?P<id4>[A-Z0-9_]*[A-Z][A-Z0-9_]*)\s*,
+    ''', re.MULTILINE | re.VERBOSE)
+  
+  def Parse(self):
+    '''Knows how to parse dialog resource sections.'''
+    self._RegExpParse(self.dialog_re_, self.text_)
+  
+  # static method
+  def FromFile(rc_file, extkey, encoding = 'cp1252'):
+    return Section.FromFileImpl(rc_file, extkey, encoding, Dialog)
+  FromFile = staticmethod(FromFile)
+
+
+class Menu(Section):
+  '''A resource section that contains a menu resource.'''
+  
+  # A typical menu resource section looks something like this:
+  #
+  # IDC_KLONK MENU 
+  # BEGIN
+  #     POPUP "&File"
+  #     BEGIN
+  #         MENUITEM "E&xit",                       IDM_EXIT
+  #         MENUITEM "This be ""Klonk"" me like",   ID_FILE_THISBE
+  #         POPUP "gonk"
+  #         BEGIN
+  #             MENUITEM "Klonk && is ""good""",           ID_GONK_KLONKIS
+  #         END
+  #     END
+  #     POPUP "&Help"
+  #     BEGIN
+  #         MENUITEM "&About ...",                  IDM_ABOUT
+  #     END
+  # END
+
+  # Description used for the messages generated for menus, to explain to
+  # the translators how to handle them.
+  MENU_MESSAGE_DESCRIPTION = (
+    'This message represents a menu. Each of the items appears in sequence '
+    '(some possibly within sub-menus) in the menu. The XX01XX placeholders '
+    'serve to separate items. Each item contains an & (ampersand) character '
+    'in front of the keystroke that should be used as a shortcut for that item '
+    'in the menu. Please make sure that no two items in the same menu share '
+    'the same shortcut.'
+  )
+  
+  # A dandy regexp to suck all the IDs and translateables out of a menu
+  # resource
+  menu_re_ = re.compile('''
+    # Match the MENU ID on the first line
+    ^(?P<id1>[A-Z0-9_]+)\s+MENU
+    |
+    # Match the translateable caption for a popup menu
+    POPUP\s+"(?P<text1>.*?([^"]|""))"\s
+    |
+    # Match the caption & ID of a MENUITEM
+    MENUITEM\s+"(?P<text2>.*?([^"]|""))"\s*,\s*(?P<id2>[A-Z0-9_]+)
+    ''', re.MULTILINE | re.VERBOSE)
+  
+  def Parse(self):
+    '''Knows how to parse menu resource sections.  Because it is important that
+    menu shortcuts are unique within the menu, we return each menu as a single
+    message with placeholders to break up the different menu items, rather than
+    return a single message per menu item.  we also add an automatic description
+    with instructions for the translators.'''
+    self.single_message_ = tclib.Message(description=self.MENU_MESSAGE_DESCRIPTION)
+    self._RegExpParse(self.menu_re_, self.text_)
+  
+  # static method
+  def FromFile(rc_file, extkey, encoding = 'cp1252'):
+    return Section.FromFileImpl(rc_file, extkey, encoding, Menu)
+  FromFile = staticmethod(FromFile)
+
+
+class Version(Section):
+  '''A resource section that contains a VERSIONINFO resource.'''
+  
+  # A typical version info resource can look like this:
+  #
+  # VS_VERSION_INFO VERSIONINFO
+  #  FILEVERSION 1,0,0,1
+  #  PRODUCTVERSION 1,0,0,1
+  #  FILEFLAGSMASK 0x3fL
+  # #ifdef _DEBUG
+  #  FILEFLAGS 0x1L
+  # #else
+  #  FILEFLAGS 0x0L
+  # #endif
+  #  FILEOS 0x4L
+  #  FILETYPE 0x2L
+  #  FILESUBTYPE 0x0L
+  # BEGIN
+  #     BLOCK "StringFileInfo"
+  #     BEGIN
+  #         BLOCK "040904e4"
+  #         BEGIN
+  #             VALUE "CompanyName", "TODO: <Company name>"
+  #             VALUE "FileDescription", "TODO: <File description>"
+  #             VALUE "FileVersion", "1.0.0.1"
+  #             VALUE "LegalCopyright", "TODO: (c) <Company name>.  All rights reserved."
+  #             VALUE "InternalName", "res_format_test.dll"
+  #             VALUE "OriginalFilename", "res_format_test.dll"
+  #             VALUE "ProductName", "TODO: <Product name>"
+  #             VALUE "ProductVersion", "1.0.0.1"
+  #         END
+  #     END
+  #     BLOCK "VarFileInfo"
+  #     BEGIN
+  #         VALUE "Translation", 0x409, 1252
+  #     END
+  # END
+  #
+  #
+  # In addition to the above fields, VALUE fields named "Comments" and
+  # "LegalTrademarks" may also be translateable.
+  
+  version_re_ = re.compile('''
+    # Match the ID on the first line
+    ^(?P<id1>[A-Z0-9_]+)\s+VERSIONINFO
+    |
+    # Match all potentially translateable VALUE sections
+    \s+VALUE\s+"
+    (
+      CompanyName|FileDescription|LegalCopyright|
+      ProductName|Comments|LegalTrademarks
+    )",\s+"(?P<text1>.*?([^"]|""))"\s
+    ''', re.MULTILINE | re.VERBOSE)
+  
+  def Parse(self):
+    '''Knows how to parse VERSIONINFO resource sections.'''
+    self._RegExpParse(self.version_re_, self.text_)
+
+  # TODO(joi) May need to override the Translate() method to change the
+  # "Translation" VALUE block to indicate the correct language code.
+
+  # static method
+  def FromFile(rc_file, extkey, encoding = 'cp1252'):
+    return Section.FromFileImpl(rc_file, extkey, encoding, Version)
+  FromFile = staticmethod(FromFile)
+  
+class RCData(Section):
+  '''A resource section that contains some data .'''
+
+  # A typical rcdataresource section looks like this:
+  #
+  # IDR_BLAH        RCDATA      { 1, 2, 3, 4 }
+  
+  dialog_re_ = re.compile('''
+    ^(?P<id1>[A-Z0-9_]+)\s+RCDATA\s+(DISCARDABLE)?\s+\{.*?\}
+    ''', re.MULTILINE | re.VERBOSE | re.DOTALL)
+  
+  def Parse(self):
+    '''Knows how to parse RCDATA resource sections.'''
+    self._RegExpParse(self.dialog_re_, self.text_)
+
+  # static method
+  def FromFile(rc_file, extkey, encoding = 'cp1252'):
+    '''Implementation of FromFile for resource types w/braces (not BEGIN/END)
+    '''
+    if isinstance(rc_file, types.StringTypes):
+      rc_file = util.WrapInputStream(file(rc_file, 'r'), encoding)
+    
+    out = ''
+    begin_count = 0
+    openbrace_count = 0
+    for line in rc_file.readlines():
+      if len(out) > 0 or line.strip().startswith(extkey):
+        out += line
+      
+      # we stop once balance the braces (could happen on one line)
+      begin_count_was = begin_count
+      if len(out) > 0:
+        openbrace_count += line.count('{')
+        begin_count += line.count('{')
+        begin_count -= line.count('}')
+      if ((begin_count_was == 1 and begin_count == 0) or
+         (openbrace_count > 0 and begin_count == 0)):
+        break
+    
+    if len(out) == 0:
+      raise exception.SectionNotFound('%s in file %s' % (extkey, rc_file))
+    
+    return RCData(out)
+  FromFile = staticmethod(FromFile)
+
+
+class Accelerators(Section):
+  '''An ACCELERATORS table.
+  '''
+  
+  # A typical ACCELERATORS section looks like this:
+  #
+  # IDR_ACCELERATOR1 ACCELERATORS
+  # BEGIN
+  #   "^C",           ID_ACCELERATOR32770,    ASCII,  NOINVERT
+  #   "^V",           ID_ACCELERATOR32771,    ASCII,  NOINVERT
+  #   VK_INSERT,      ID_ACCELERATOR32772,    VIRTKEY, CONTROL, NOINVERT
+  # END
+  
+  accelerators_re_ = re.compile('''
+    # Match the ID on the first line
+    ^(?P<id1>[A-Z0-9_]+)\s+ACCELERATORS\s+
+    |
+    # Match accelerators specified as VK_XXX
+    \s+VK_[A-Z0-9_]+,\s*(?P<id2>[A-Z0-9_]+)\s*,
+    |
+    # Match accelerators specified as e.g. "^C"
+    \s+"[^"]*",\s+(?P<id3>[A-Z0-9_]+)\s*,
+    ''', re.MULTILINE | re.VERBOSE)
+  
+  def Parse(self):
+    '''Knows how to parse ACCELERATORS resource sections.'''
+    self._RegExpParse(self.accelerators_re_, self.text_)
+  
+  # static method
+  def FromFile(rc_file, extkey, encoding = 'cp1252'):
+    return Section.FromFileImpl(rc_file, extkey, encoding, Accelerators)
+  FromFile = staticmethod(FromFile)

tools/grit/grit/gather/rc_unittest.py

@@ -0,0 +1,390 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.gather.rc'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import unittest
+import StringIO
+
+from grit.gather import rc
+from grit import util
+
+
+class RcUnittest(unittest.TestCase):
+
+  part_we_want = '''IDC_KLONKACC ACCELERATORS 
+BEGIN
+    "?",            IDM_ABOUT,              ASCII,  ALT
+    "/",            IDM_ABOUT,              ASCII,  ALT
+END'''
+
+  def testSectionFromFile(self):
+    buf = '''IDC_SOMETHINGELSE BINGO
+BEGIN
+    BLA BLA
+    BLA BLA
+END
+%s
+
+IDC_KLONK BINGOBONGO
+BEGIN
+  HONGO KONGO
+END
+''' % self.part_we_want
+
+    f = StringIO.StringIO(buf)
+    
+    out = rc.Section.FromFile(f, 'IDC_KLONKACC')
+    self.failUnless(out.GetText() == self.part_we_want)
+
+    out = rc.Section.FromFile(util.PathFromRoot(r'grit/test/data/klonk.rc'),
+                              'IDC_KLONKACC',
+                              encoding='utf-16')
+    self.failUnless(out.GetText() == self.part_we_want)
+
+
+  def testDialog(self):
+    dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75
+STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU
+CAPTION "About"
+FONT 8, "System", 0, 0, 0x0
+BEGIN
+    ICON            IDI_KLONK,IDC_MYICON,14,9,20,20
+    LTEXT           "klonk Version ""yibbee"" 1.0",IDC_STATIC,49,10,119,8,
+                    SS_NOPREFIX
+    LTEXT           "Copyright (C) 2005",IDC_STATIC,49,20,119,8
+    DEFPUSHBUTTON   "OK",IDOK,195,6,30,11,WS_GROUP
+    CONTROL         "Jack ""Black"" Daniels",IDC_RADIO1,"Button",
+                    BS_AUTORADIOBUTTON,46,51,84,10
+    // try a line where the ID is on the continuation line
+    LTEXT           "blablablabla blablabla blablablablablablablabla blablabla",
+                    ID_SMURF, whatever...
+END
+''')
+    dlg.Parse()
+    self.failUnless(len(dlg.GetTextualIds()) == 7)
+    self.failUnless(len(dlg.GetCliques()) == 6)
+    self.failUnless(dlg.GetCliques()[1].GetMessage().GetRealContent() ==
+                    'klonk Version "yibbee" 1.0')
+    
+    transl = dlg.Translate('en')
+    self.failUnless(transl.strip() == dlg.GetText().strip())
+
+  def testAlternateSkeleton(self):
+    dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 22, 17, 230, 75
+STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU
+CAPTION "About"
+FONT 8, "System", 0, 0, 0x0
+BEGIN
+    LTEXT           "Yipee skippy",IDC_STATIC,49,10,119,8,
+                    SS_NOPREFIX
+END
+''')
+    dlg.Parse()
+    
+    alt_dlg = rc.Dialog('''IDD_ABOUTBOX DIALOGEX 040704, 17, 230, 75
+STYLE DS_SETFONT | DS_MODALFRAME | WS_CAPTION | WS_SYSMENU
+CAPTION "XXXXXXXXX"
+FONT 8, "System", 0, 0, 0x0
+BEGIN
+    LTEXT           "XXXXXXXXXXXXXXXXX",IDC_STATIC,110978,10,119,8,
+                    SS_NOPREFIX
+END
+''')
+    alt_dlg.Parse()
+    
+    transl = dlg.Translate('en', skeleton_gatherer=alt_dlg)
+    self.failUnless(transl.count('040704') and
+                    transl.count('110978'))
+    self.failUnless(transl.count('Yipee skippy'))
+
+  def testMenu(self):
+    menu = rc.Menu('''IDC_KLONK MENU 
+BEGIN
+    POPUP "&File """
+    BEGIN
+        MENUITEM "E&xit",                       IDM_EXIT
+        MENUITEM "This be ""Klonk"" me like",   ID_FILE_THISBE
+        POPUP "gonk"
+        BEGIN
+            MENUITEM "Klonk && is ""good""",           ID_GONK_KLONKIS
+        END
+        MENUITEM "This is a very long menu caption to try to see if we can make the ID go to a continuation line, blablabla blablabla bla blabla blablabla blablabla blablabla blablabla...", 
+                                        ID_FILE_THISISAVERYLONGMENUCAPTIONTOTRYTOSEEIFWECANMAKETHEIDGOTOACONTINUATIONLINE
+    END
+    POPUP "&Help"
+    BEGIN
+        MENUITEM "&About ...",                  IDM_ABOUT
+    END
+END''')
+    
+    menu.Parse()
+    self.failUnless(len(menu.GetTextualIds()) == 6)
+    self.failUnless(len(menu.GetCliques()) == 1)
+    self.failUnless(len(menu.GetCliques()[0].GetMessage().GetPlaceholders()) ==
+                    9)
+    
+    transl = menu.Translate('en')
+    self.failUnless(transl.strip() == menu.GetText().strip())
+
+  def testVersion(self):
+    version = rc.Version('''
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION 1,0,0,1
+ PRODUCTVERSION 1,0,0,1
+ FILEFLAGSMASK 0x3fL
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x2L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904e4"
+        BEGIN
+            VALUE "CompanyName", "TODO: <Company name>"
+            VALUE "FileDescription", "TODO: <File description>"
+            VALUE "FileVersion", "1.0.0.1"
+            VALUE "LegalCopyright", "TODO: (c) <Company name>.  All rights reserved."
+            VALUE "InternalName", "res_format_test.dll"
+            VALUE "OriginalFilename", "res_format_test.dll"
+            VALUE "ProductName", "TODO: <Product name>"
+            VALUE "ProductVersion", "1.0.0.1"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1252
+    END
+END
+'''.strip())
+    version.Parse()
+    self.failUnless(len(version.GetTextualIds()) == 1)
+    self.failUnless(len(version.GetCliques()) == 4)
+    
+    transl = version.Translate('en')
+    self.failUnless(transl.strip() == version.GetText().strip())
+
+
+  def testRegressionDialogBox(self):
+    dialog = rc.Dialog('''
+IDD_SIDEBAR_WEATHER_PANEL_PROPPAGE DIALOGEX 0, 0, 205, 157
+STYLE DS_SETFONT | DS_FIXEDSYS | WS_CHILD
+FONT 8, "MS Shell Dlg", 400, 0, 0x1
+BEGIN
+    EDITTEXT        IDC_SIDEBAR_WEATHER_NEW_CITY,3,27,112,14,ES_AUTOHSCROLL
+    DEFPUSHBUTTON   "Add Location",IDC_SIDEBAR_WEATHER_ADD,119,27,50,14
+    LISTBOX         IDC_SIDEBAR_WEATHER_CURR_CITIES,3,48,127,89,
+                    LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_TABSTOP
+    PUSHBUTTON      "Move Up",IDC_SIDEBAR_WEATHER_MOVE_UP,134,104,50,14
+    PUSHBUTTON      "Move Down",IDC_SIDEBAR_WEATHER_MOVE_DOWN,134,121,50,14
+    PUSHBUTTON      "Remove",IDC_SIDEBAR_WEATHER_DELETE,134,48,50,14
+    LTEXT           "To see current weather conditions and forecasts in the USA, enter the zip code (example: 94043) or city and state (example: Mountain View, CA).",
+                    IDC_STATIC,3,0,199,25
+    CONTROL         "Fahrenheit",IDC_SIDEBAR_WEATHER_FAHRENHEIT,"Button",
+                    BS_AUTORADIOBUTTON | WS_GROUP | WS_TABSTOP,3,144,51,10
+    CONTROL         "Celsius",IDC_SIDEBAR_WEATHER_CELSIUS,"Button",
+                    BS_AUTORADIOBUTTON,57,144,38,10
+END'''.strip())
+    dialog.Parse()
+    self.failUnless(len(dialog.GetTextualIds()) == 10)
+  
+  
+  def testRegressionDialogBox2(self):
+    dialog = rc.Dialog('''
+IDD_SIDEBAR_EMAIL_PANEL_PROPPAGE DIALOG DISCARDABLE 0, 0, 264, 220
+STYLE WS_CHILD
+FONT 8, "MS Shell Dlg"
+BEGIN
+    GROUPBOX        "Email Filters",IDC_STATIC,7,3,250,190
+    LTEXT           "Click Add Filter to create the email filter.",IDC_STATIC,16,41,130,9
+    PUSHBUTTON      "Add Filter...",IDC_SIDEBAR_EMAIL_ADD_FILTER,196,38,50,14
+    PUSHBUTTON      "Remove",IDC_SIDEBAR_EMAIL_REMOVE,196,174,50,14
+    PUSHBUTTON      "", IDC_SIDEBAR_EMAIL_HIDDEN, 200, 178, 5, 5, NOT WS_VISIBLE
+    LISTBOX         IDC_SIDEBAR_EMAIL_LIST,16,60,230,108, 
+                    LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_TABSTOP
+    LTEXT           "You can prevent certain emails from showing up in the sidebar with a filter.",
+                    IDC_STATIC,16,18,234,18
+END'''.strip())
+    dialog.Parse()
+    self.failUnless('IDC_SIDEBAR_EMAIL_HIDDEN' in dialog.GetTextualIds())
+
+  
+  def testRegressionMenuId(self):
+    menu = rc.Menu('''
+IDR_HYPERMENU_FOLDER MENU
+BEGIN
+    POPUP "HyperFolder"
+    BEGIN
+        MENUITEM "Open Containing Folder",      IDM_OPENFOLDER
+    END
+END'''.strip())
+    menu.Parse()
+    self.failUnless(len(menu.GetTextualIds()) == 2)
+
+  def testRegressionNewlines(self):
+    menu = rc.Menu('''
+IDR_HYPERMENU_FOLDER MENU
+BEGIN
+    POPUP "Hyper\\nFolder"
+    BEGIN
+        MENUITEM "Open Containing Folder",      IDM_OPENFOLDER
+    END
+END'''.strip())
+    menu.Parse()
+    transl = menu.Translate('en')
+    # Shouldn't find \\n (the \n shouldn't be changed to \\n)
+    self.failUnless(transl.find('\\\\n') == -1)
+  
+  def testRegressionTabs(self):
+    menu = rc.Menu('''
+IDR_HYPERMENU_FOLDER MENU
+BEGIN
+    POPUP "Hyper\\tFolder"
+    BEGIN
+        MENUITEM "Open Containing Folder",      IDM_OPENFOLDER
+    END
+END'''.strip())
+    menu.Parse()
+    transl = menu.Translate('en')
+    # Shouldn't find \\t (the \t shouldn't be changed to \\t)
+    self.failUnless(transl.find('\\\\t') == -1)
+
+  def testEscapeUnescape(self):
+    original = 'Hello "bingo"\n How\\are\\you\\n?'
+    escaped = rc.Section.Escape(original)
+    self.failUnless(escaped == 'Hello ""bingo""\\n How\\\\are\\\\you\\\\n?')
+    unescaped = rc.Section.UnEscape(escaped)
+    self.failUnless(unescaped == original)
+
+  def testRegressionPathsWithSlashN(self):
+    original = '..\\\\..\\\\trs\\\\res\\\\nav_first.gif'
+    unescaped = rc.Section.UnEscape(original)
+    self.failUnless(unescaped == '..\\..\\trs\\res\\nav_first.gif')
+  
+  def testRegressionDialogItemsTextOnly(self):
+    dialog = rc.Dialog('''IDD_OPTIONS_SEARCH DIALOGEX 0, 0, 280, 292
+STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP |
+    WS_DISABLED | WS_CAPTION | WS_SYSMENU
+CAPTION "Search"
+FONT 8, "MS Shell Dlg", 400, 0, 0x1
+BEGIN
+    GROUPBOX        "Select search buttons and options",-1,7,5,266,262
+    CONTROL         "",IDC_OPTIONS,"SysTreeView32",TVS_DISABLEDRAGDROP |
+                    WS_BORDER | WS_TABSTOP | 0x800,16,19,248,218
+    LTEXT           "Use Google site:",-1,26,248,52,8
+    COMBOBOX        IDC_GOOGLE_HOME,87,245,177,256,CBS_DROPDOWNLIST |
+                    WS_VSCROLL | WS_TABSTOP
+    PUSHBUTTON      "Restore Defaults...",IDC_RESET,187,272,86,14
+END''')
+    dialog.Parse()
+    translateables = [c.GetMessage().GetRealContent()
+                      for c in dialog.GetCliques()]
+    self.failUnless('Select search buttons and options' in translateables)
+    self.failUnless('Use Google site:' in translateables)
+  
+  def testAccelerators(self):
+    acc = rc.Accelerators('''\
+IDR_ACCELERATOR1 ACCELERATORS
+BEGIN
+    "^C",           ID_ACCELERATOR32770,    ASCII,  NOINVERT
+    "^V",           ID_ACCELERATOR32771,    ASCII,  NOINVERT
+    VK_INSERT,      ID_ACCELERATOR32772,    VIRTKEY, CONTROL, NOINVERT
+END
+''')
+    acc.Parse()
+    self.failUnless(len(acc.GetTextualIds()) == 4)
+    self.failUnless(len(acc.GetCliques()) == 0)
+
+    transl = acc.Translate('en')
+    self.failUnless(transl.strip() == acc.GetText().strip())
+
+
+  def testRegressionEmptyString(self):
+    dlg = rc.Dialog('''\
+IDD_CONFIRM_QUIT_GD_DLG DIALOGEX 0, 0, 267, 108
+STYLE DS_SETFONT | DS_MODALFRAME | DS_FIXEDSYS | DS_CENTER | WS_POPUP | 
+    WS_CAPTION
+EXSTYLE WS_EX_TOPMOST
+CAPTION "Google Desktop"
+FONT 8, "MS Shell Dlg", 400, 0, 0x1
+BEGIN
+    DEFPUSHBUTTON   "&Yes",IDYES,82,87,50,14
+    PUSHBUTTON      "&No",IDNO,136,87,50,14
+    ICON            32514,IDC_STATIC,7,9,21,20
+    EDITTEXT        IDC_TEXTBOX,34,7,231,60,ES_MULTILINE | ES_READONLY | NOT WS_BORDER
+    CONTROL         "",
+                    IDC_ENABLE_GD_AUTOSTART,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,33,70,231,10
+END''')
+    dlg.Parse()
+    
+    def Check():
+      self.failUnless(transl.count('IDC_ENABLE_GD_AUTOSTART'))
+      self.failUnless(transl.count('END'))
+    
+    transl = dlg.Translate('de', pseudo_if_not_available=True,
+                           fallback_to_english=True)
+    Check()
+    transl = dlg.Translate('de', pseudo_if_not_available=True,
+                           fallback_to_english=False)
+    Check()
+    transl = dlg.Translate('de', pseudo_if_not_available=False,
+                           fallback_to_english=True)
+    Check()
+    transl = dlg.Translate('de', pseudo_if_not_available=False,
+                           fallback_to_english=False)
+    Check()
+    transl = dlg.Translate('en', pseudo_if_not_available=True,
+                           fallback_to_english=True)
+    Check()
+    transl = dlg.Translate('en', pseudo_if_not_available=True,
+                           fallback_to_english=False)
+    Check()
+    transl = dlg.Translate('en', pseudo_if_not_available=False,
+                           fallback_to_english=True)
+    Check()
+    transl = dlg.Translate('en', pseudo_if_not_available=False,
+                           fallback_to_english=False)
+    Check()
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/gather/regexp.py

@@ -0,0 +1,224 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''A baseclass for simple gatherers based on regular expressions.
+'''
+
+import re
+import types
+
+from grit.gather import interface
+from grit import clique
+from grit import tclib
+
+
+class RegexpGatherer(interface.GathererBase):
+  '''Common functionality of gatherers based on parsing using a single
+  regular expression.
+  '''
+
+  DescriptionMapping_ = {
+      'CAPTION' : 'This is a caption for a dialog',
+      'CHECKBOX' : 'This is a label for a checkbox',
+      'CONTROL': 'This is the text on a control',
+      'CTEXT': 'This is a label for a control',
+      'DEFPUSHBUTTON': 'This is a button definition',
+      'GROUPBOX': 'This is a label for a grouping',
+      'ICON': 'This is a label for an icon',
+      'LTEXT': 'This is the text for a label',
+      'PUSHBUTTON': 'This is the text for a button',
+    }
+  
+  def __init__(self, text):
+    interface.GathererBase.__init__(self)
+    # Original text of what we're parsing
+    self.text_ = text.strip()
+    # List of parts of the document. Translateable parts are clique.MessageClique
+    # objects, nontranslateable parts are plain strings. Translated messages are
+    # inserted back into the skeleton using the quoting rules defined by
+    # self.Escape()
+    self.skeleton_ = []
+    # A list of the names of IDs that need to be defined for this resource
+    # section to compile correctly.
+    self.ids_ = []
+    # True if Parse() has already been called.
+    self.have_parsed_ = False
+    # True if a translatable chunk has been added
+    self.translatable_chunk_ = False
+    # If not None, all parts of the document will be put into this single
+    # message; otherwise the normal skeleton approach is used.
+    self.single_message_ = None
+    # Number to use for the next placeholder name.  Used only if single_message
+    # is not None
+    self.ph_counter_ = 1
+    
+  def GetText(self):
+    '''Returns the original text of the section'''
+    return self.text_
+  
+  def Escape(self, text):
+    '''Subclasses can override.  Base impl is identity.
+    '''
+    return text
+  
+  def UnEscape(self, text):
+    '''Subclasses can override. Base impl is identity.
+    '''
+    return text
+  
+  def GetTextualIds(self):
+    '''Returns the list of textual IDs that need to be defined for this
+    resource section to compile correctly.'''
+    return self.ids_
+  
+  def GetCliques(self):
+    '''Returns the message cliques for each translateable message in the
+    resource section.'''
+    return filter(lambda x: isinstance(x, clique.MessageClique), self.skeleton_)
+  
+  def Translate(self, lang, pseudo_if_not_available=True,
+                skeleton_gatherer=None, fallback_to_english=False):
+    if len(self.skeleton_) == 0:
+      raise exception.NotReady()
+    if skeleton_gatherer:
+      assert len(skeleton_gatherer.skeleton_) == len(self.skeleton_)
+    
+    out = []
+    for ix in range(len(self.skeleton_)):
+      if isinstance(self.skeleton_[ix], types.StringTypes):
+        if skeleton_gatherer:
+          # Make sure the skeleton is like the original
+          assert(isinstance(skeleton_gatherer.skeleton_[ix], types.StringTypes))
+          out.append(skeleton_gatherer.skeleton_[ix])
+        else:
+          out.append(self.skeleton_[ix])
+      else:
+        if skeleton_gatherer:  # Make sure the skeleton is like the original
+          assert(not isinstance(skeleton_gatherer.skeleton_[ix],
+                                types.StringTypes))
+        msg = self.skeleton_[ix].MessageForLanguage(lang,
+                                                    pseudo_if_not_available,
+                                                    fallback_to_english)
+        
+        def MyEscape(text):
+          return self.Escape(text)
+        text = msg.GetRealContent(escaping_function=MyEscape)
+        out.append(text)
+    return ''.join(out)
+
+  # Contextualization elements. Used for adding additional information
+  # to the message bundle description string from RC files.
+  def AddDescriptionElement(self, string):
+    if self.DescriptionMapping_.has_key(string):
+      description = self.DescriptionMapping_[string]
+    else:
+      description = string
+    if self.single_message_:
+      self.single_message_.SetDescription(description)
+    else:
+      if (self.translatable_chunk_):
+        message = self.skeleton_[len(self.skeleton_) - 1].GetMessage()
+        message.SetDescription(description)
+  
+  def Parse(self):
+    '''Parses the section.  Implemented by subclasses.  Idempotent.'''
+    raise NotImplementedError()
+
+  def _AddNontranslateableChunk(self, chunk):
+    '''Adds a nontranslateable chunk.'''
+    if self.single_message_:
+      ph = tclib.Placeholder('XX%02dXX' % self.ph_counter_, chunk, chunk)
+      self.ph_counter_ += 1
+      self.single_message_.AppendPlaceholder(ph)
+    else:
+      self.skeleton_.append(chunk)
+  
+  def _AddTranslateableChunk(self, chunk):
+    '''Adds a translateable chunk.  It will be unescaped before being added.'''
+    # We don't want empty messages since they are redundant and the TC
+    # doesn't allow them.
+    if chunk == '':
+      return
+    
+    unescaped_text = self.UnEscape(chunk)
+    if self.single_message_:
+      self.single_message_.AppendText(unescaped_text)
+    else:
+      self.skeleton_.append(self.uberclique.MakeClique(
+        tclib.Message(text=unescaped_text)))
+      self.translatable_chunk_ = True
+  
+  def _AddTextualId(self, id):
+    self.ids_.append(id)
+  
+  def _RegExpParse(self, regexp, text_to_parse):
+    '''An implementation of Parse() that can be used for resource sections that
+    can be parsed using a single multi-line regular expression.
+    
+    All translateables must be in named groups that have names starting with
+    'text'.  All textual IDs must be in named groups that have names starting
+    with 'id'. All type definitions that can be included in the description
+    field for contextualization purposes should have a name that starts with
+    'type'.
+    
+    Args:
+      regexp: re.compile('...', re.MULTILINE)
+      text_to_parse: 
+    '''
+    if self.have_parsed_:
+      return
+    self.have_parsed_ = True
+
+    chunk_start = 0
+    for match in regexp.finditer(text_to_parse):
+      groups = match.groupdict()
+      keys = groups.keys()
+      keys.sort()
+      self.translatable_chunk_ = False
+      for group in keys:
+        if group.startswith('id') and groups[group]:
+          self._AddTextualId(groups[group])
+        elif group.startswith('text') and groups[group]:
+          self._AddNontranslateableChunk(
+            text_to_parse[chunk_start : match.start(group)])
+          chunk_start = match.end(group)  # Next chunk will start after the match
+          self._AddTranslateableChunk(groups[group])
+        elif group.startswith('type') and groups[group]:
+          # Add the description to the skeleton_ list. This works because
+          # we are using a sort set of keys, and because we assume that the
+          # group name used for descriptions (type) will come after the "text"
+          # group in alphabetical order. We also assume that there cannot be
+          # more than one description per regular expression match.
+          self.AddDescriptionElement(groups[group])
+    
+    self._AddNontranslateableChunk(text_to_parse[chunk_start:])
+    
+    if self.single_message_:
+      self.skeleton_.append(self.uberclique.MakeClique(self.single_message_))

tools/grit/grit/gather/tr_html.py

@@ -0,0 +1,703 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''A gatherer for the TotalRecall brand of HTML templates with replaceable
+portions.  We wanted to reuse extern.tclib.api.handlers.html.TCHTMLParser
+but this proved impossible due to the fact that the TotalRecall HTML templates
+are in general quite far from parseable HTML and the TCHTMLParser derives
+from HTMLParser.HTMLParser which requires relatively well-formed HTML.  Some
+examples of "HTML" from the TotalRecall HTML templates that wouldn't be
+parseable include things like:
+
+  <a [PARAMS]>blabla</a>  (not parseable because attributes are invalid)
+
+  <table><tr><td>[LOTSOFSTUFF]</tr></table> (not parseable because closing
+                                            </td> is in the HTML [LOTSOFSTUFF]
+                                            is replaced by)
+
+The other problem with using general parsers (such as TCHTMLParser) is that
+we want to make sure we output the TotalRecall template with as little changes
+as possible in terms of whitespace characters, layout etc.  With any parser
+that generates a parse tree, and generates output by dumping the parse tree,
+we would always have little inconsistencies which could cause bugs (the
+TotalRecall template stuff is quite brittle and can break if e.g. a tab
+character is replaced with spaces).
+
+The solution, which may be applicable to some other HTML-like template
+languages floating around Google, is to create a parser with a simple state
+machine that keeps track of what kind of tag it's inside, and whether it's in
+a translateable section or not.  Translateable sections are:
+
+a) text (including [BINGO] replaceables) inside of tags that
+   can contain translateable text (which is all tags except
+   for a few)
+
+b) text inside of an 'alt' attribute in an <image> element, or
+   the 'value' attribute of a <submit>, <button> or <text>
+   element.
+
+The parser does not build up a parse tree but rather a "skeleton" which
+is a list of nontranslateable strings intermingled with grit.clique.MessageClique
+objects.  This simplifies the parser considerably compared to a regular HTML
+parser.  To output a translated document, each item in the skeleton is
+printed out, with the relevant Translation from each MessageCliques being used
+for the requested language.
+
+This implementation borrows some code, constants and ideas from
+extern.tclib.api.handlers.html.TCHTMLParser.
+'''
+
+
+import re
+import types
+
+from grit import clique
+from grit import exception
+from grit import util
+from grit import tclib
+
+from grit.gather import interface
+
+
+# HTML tags which break (separate) chunks.
+_BLOCK_TAGS = ['script', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'br',
+              'body', 'style', 'head', 'title', 'table', 'tr', 'td', 'th',
+              'ul', 'ol', 'dl', 'nl', 'li', 'div', 'object', 'center',
+              'html', 'link', 'form', 'select', 'textarea',
+              'button', 'option', 'map', 'area', 'blockquote', 'pre',
+              'meta', 'xmp', 'noscript', 'label', 'tbody', 'thead',
+              'script', 'style', 'pre', 'iframe', 'img', 'input', 'nowrap']
+
+# HTML tags which may appear within a chunk.
+_INLINE_TAGS = ['b', 'i', 'u', 'tt', 'code', 'font', 'a', 'span', 'small',
+               'key', 'nobr', 'url', 'em', 's', 'sup', 'strike',
+               'strong']
+
+# HTML tags within which linebreaks are significant.
+_PREFORMATTED_TAGS = ['textarea', 'xmp', 'pre']
+
+# An array mapping some of the inline HTML tags to more meaningful
+# names for those tags.  This will be used when generating placeholders
+# representing these tags.
+_HTML_PLACEHOLDER_NAMES = { 'a' : 'link', 'br' : 'break', 'b' : 'bold',
+  'i' : 'italic', 'li' : 'item', 'ol' : 'ordered_list', 'p' : 'paragraph',
+  'ul' : 'unordered_list', 'img' : 'image', 'em' : 'emphasis' }
+
+# We append each of these characters in sequence to distinguish between
+# different placeholders with basically the same name (e.g. BOLD1, BOLD2).
+# Keep in mind that a placeholder name must not be a substring of any other
+# placeholder name in the same message, so we can't simply count (BOLD_1
+# would be a substring of BOLD_10).
+_SUFFIXES = '123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+# Matches whitespace in an HTML document.  Also matches HTML comments, which are
+# treated as whitespace.
+_WHITESPACE = re.compile(r'(\s|&nbsp;|\\n|\\r|<!--\s*desc\s*=.*?-->)+',
+                         re.DOTALL)
+
+# Finds a non-whitespace character
+_NON_WHITESPACE = re.compile(r'\S')
+
+# Matches two or more &nbsp; in a row (a single &nbsp is not changed into
+# placeholders because different languages require different numbers of spaces
+# and placeholders must match exactly; more than one is probably a "special"
+# whitespace sequence and should be turned into a placeholder).
+_NBSP = re.compile(r'&nbsp;(&nbsp;)+')
+
+# Matches nontranslateable chunks of the document
+_NONTRANSLATEABLES = re.compile(r'''
+  <\s*script.+?<\s*/\s*script\s*>
+  |
+  <\s*style.+?<\s*/\s*style\s*>
+  |
+  <!--.+?-->
+  |
+  <\?IMPORT\s.+?>           # import tag
+  |
+  <\s*[a-zA-Z_]+:.+?>       # custom tag (open)
+  |
+  <\s*/\s*[a-zA-Z_]+:.+?>   # custom tag (close)  
+  |
+  <!\s*[A-Z]+\s*([^>]+|"[^"]+"|'[^']+')*?>
+  ''', re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE)
+
+# Matches a tag and its attributes
+_ELEMENT = re.compile(r'''
+  # Optional closing /, element name
+  <\s*(?P<closing>/)?\s*(?P<element>[a-zA-Z0-9]+)\s*
+  # Attributes and/or replaceables inside the tag, if any
+  (?P<atts>(
+    \s*([a-zA-Z_][-:.a-zA-Z_0-9]*) # Attribute name
+    (\s*=\s*(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?
+    |
+    \s*\[(\$?\~)?([A-Z0-9-_]+?)(\~\$?)?\]
+  )*)
+  \s*(?P<empty>/)?\s*> # Optional empty-tag closing /, and tag close
+  ''',
+  re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+# Matches elements that may have translateable attributes.  The value of these
+# special attributes is given by group 'value1' or 'value2'.  Note that this
+# regexp demands that the attribute value be quoted; this is necessary because
+# the non-tree-building nature of the parser means we don't know when we're
+# writing out attributes, so we wouldn't know to escape spaces.
+_SPECIAL_ELEMENT = re.compile(r'''
+  <\s*(
+    input[^>]+?value\s*=\s*(\'(?P<value3>[^\']*)\'|"(?P<value4>[^"]*)")
+    [^>]+type\s*=\s*"?'?(button|reset|text|submit)'?"?
+    |
+    (
+      table[^>]+?title\s*=
+      |
+      img[^>]+?alt\s*=
+      |
+      input[^>]+?type\s*=\s*"?'?(button|reset|text|submit)'?"?[^>]+?value\s*=
+    )
+    \s*(\'(?P<value1>[^\']*)\'|"(?P<value2>[^"]*)")
+  )[^>]*?>
+  ''', re.MULTILINE | re.DOTALL | re.VERBOSE | re.IGNORECASE)
+
+# Matches stuff that is translateable if it occurs in the right context
+# (between tags).  This includes all characters and character entities.
+# Note that this also matches &nbsp; which needs to be handled as whitespace
+# before this regexp is applied.
+_CHARACTERS = re.compile(r'''
+  (
+    \w
+    |
+    [\!\@\#\$\%\^\*\(\)\-\=\_\+\[\]\{\}\\\|\;\:\'\"\,\.\/\?\`\~]
+    |
+    &(\#[0-9]+|\#x[0-9a-fA-F]+|[A-Za-z0-9]+);
+  )+
+  ''', re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+# Matches Total Recall's "replaceable" tags, which are just any text
+# in capitals enclosed by delimiters like [] or [~~] or [$~~$] (e.g. [HELLO],
+# [~HELLO~] and [$~HELLO~$]).
+_REPLACEABLE = re.compile(r'\[(\$?\~)?(?P<name>[A-Z0-9-_]+?)(\~\$?)?\]',
+                          re.MULTILINE)
+
+
+# Matches the silly [!]-prefixed "header" that is used in some TotalRecall
+# templates.
+_SILLY_HEADER = re.compile(r'\[!\]\ntitle\t(?P<title>[^\n]+?)\n.+?\n\n',
+                           re.MULTILINE | re.DOTALL)
+
+
+# Matches a comment that provides a description for the message it occurs in.
+_DESCRIPTION_COMMENT = re.compile(
+  r'<!--\s*desc\s*=\s*(?P<description>.+?)\s*-->', re.DOTALL)
+
+
+_DEBUG = 0
+def _DebugPrint(text):
+  if _DEBUG:
+    print text.encode('utf-8')
+
+
+class HtmlChunks(object):
+  '''A parser that knows how to break an HTML-like document into a list of
+  chunks, where each chunk is either translateable or non-translateable.
+  The chunks are unmodified sections of the original document, so concatenating
+  the text of all chunks would result in the original document.'''
+  
+  def InTranslateable(self):
+    return self.last_translateable != -1
+  
+  def Rest(self):
+    return self.text_[self.current:]
+  
+  def StartTranslateable(self):
+    assert not self.InTranslateable()
+    if self.current != 0:
+      # Append a nontranslateable chunk
+      chunk_text = self.text_[self.chunk_start : self.last_nontranslateable + 1]
+      # Needed in the case where document starts with a translateable.
+      if len(chunk_text) > 0:
+        self.AddChunk(False, chunk_text)
+    self.chunk_start = self.last_nontranslateable + 1
+    self.last_translateable = self.current
+    self.last_nontranslateable = -1
+  
+  def EndTranslateable(self):
+    assert self.InTranslateable()
+    # Append a translateable chunk
+    self.AddChunk(True,
+                  self.text_[self.chunk_start : self.last_translateable + 1])
+    self.chunk_start = self.last_translateable + 1
+    self.last_translateable = -1
+    self.last_nontranslateable = self.current
+  
+  def AdvancePast(self, match):
+    self.current += match.end()
+  
+  def AddChunk(self, translateable, text):
+    '''Adds a chunk to self, removing linebreaks and duplicate whitespace
+    if appropriate.
+    '''
+    if translateable and not self.last_element_ in _PREFORMATTED_TAGS:
+      text = text.replace('\n', ' ')
+      text = text.replace('\r', ' ')
+      text = text.replace('   ', ' ')
+      text = text.replace('  ', ' ')
+    
+    m = _DESCRIPTION_COMMENT.search(text)
+    if m:
+      self.last_description = m.group('description')
+      # remove the description from the output text
+      text = _DESCRIPTION_COMMENT.sub('', text)
+    
+    if translateable:
+      description = self.last_description
+      self.last_description = ''
+    else:
+      description = ''
+    
+    if text != '':
+      self.chunks_.append((translateable, text, description))
+      
+  def Parse(self, text):
+    '''Parses self.text_ into an intermediate format stored in self.chunks_
+    which is translateable and nontranslateable chunks.  Also returns
+    self.chunks_
+    
+    Return:
+      [chunk1, chunk2, chunk3, ...]  (instances of class Chunk)
+    '''
+    #
+    # Chunker state
+    #
+    
+    self.text_ = text
+    
+    # A list of tuples (is_translateable, text) which represents the document
+    # after chunking.
+    self.chunks_ = []
+    
+    # Start index of the last chunk, whether translateable or not
+    self.chunk_start = 0
+    
+    # Index of the last for-sure translateable character if we are parsing
+    # a translateable chunk, -1 to indicate we are not in a translateable chunk.
+    # This is needed so that we don't include trailing whitespace in the
+    # translateable chunk (whitespace is neutral).
+    self.last_translateable = -1
+    
+    # Index of the last for-sure nontranslateable character if we are parsing
+    # a nontranslateable chunk, -1 if we are not in a nontranslateable chunk.
+    # This is needed to make sure we can group e.g. "<b>Hello</b> there"
+    # together instead of just "Hello</b> there" which would be much worse
+    # for translation.
+    self.last_nontranslateable = -1
+    
+    # Index of the character we're currently looking at.
+    self.current = 0
+    
+    # The name of the last block element parsed.
+    self.last_element_ = ''
+    
+    # The last explicit description we found.
+    self.last_description = ''
+    
+    while self.current < len(self.text_):
+      _DebugPrint('REST: %s' % self.text_[self.current:self.current+60])
+      
+      # First try to match whitespace
+      m = _WHITESPACE.match(self.Rest())
+      if m:
+        # Whitespace is neutral, it just advances 'current' and does not switch
+        # between translateable/nontranslateable.  If we are in a
+        # nontranslateable section that extends to the current point, we extend
+        # it to include the whitespace.  If we are in a translateable section,
+        # we do not extend it until we find
+        # more translateable parts, because we never want a translateable chunk
+        # to end with whitespace.
+        if (not self.InTranslateable() and
+            self.last_nontranslateable == self.current - 1):
+          self.last_nontranslateable = self.current + m.end() - 1
+        self.AdvancePast(m)
+        continue
+      
+      # Then we try to match nontranslateables
+      m = _NONTRANSLATEABLES.match(self.Rest())
+      if m:
+        if self.InTranslateable():
+          self.EndTranslateable()
+        self.last_nontranslateable = self.current + m.end() - 1
+        self.AdvancePast(m)
+        continue
+      
+      # Now match all other HTML element tags (opening, closing, or empty, we
+      # don't care).
+      m = _ELEMENT.match(self.Rest())
+      if m:
+        element_name = m.group('element').lower()
+        if element_name in _BLOCK_TAGS:
+          self.last_element_ = element_name
+          if self.InTranslateable():
+            self.EndTranslateable()
+          
+          # Check for "special" elements, i.e. ones that have a translateable
+          # attribute, and handle them correctly.  Note that all of the
+          # "special" elements are block tags, so no need to check for this
+          # if the tag is not a block tag.
+          sm = _SPECIAL_ELEMENT.match(self.Rest())
+          if sm:
+            # Get the appropriate group name
+            for group in sm.groupdict().keys():
+              if sm.groupdict()[group]:
+                break
+              
+            # First make a nontranslateable chunk up to and including the
+            # quote before the translateable attribute value
+            self.AddChunk(False, self.text_[
+              self.chunk_start : self.current + sm.start(group)])
+            # Then a translateable for the translateable bit
+            self.AddChunk(True, self.Rest()[sm.start(group) : sm.end(group)])
+            # Finally correct the data invariant for the parser
+            self.chunk_start = self.current + sm.end(group)
+          
+          self.last_nontranslateable = self.current + m.end() - 1
+        elif self.InTranslateable():
+          # We're in a translateable and the tag is an inline tag, so we
+          # need to include it in the translateable.
+          self.last_translateable = self.current + m.end() - 1
+        self.AdvancePast(m)
+        continue
+      
+      # Anything else we find must be translateable, so we advance one character
+      # at a time until one of the above matches.
+      if not self.InTranslateable():
+        self.StartTranslateable()
+      else:
+        self.last_translateable = self.current
+      self.current += 1
+    
+    # Close the final chunk
+    if self.InTranslateable():
+      self.AddChunk(True, self.text_[self.chunk_start : ])
+    else:
+      self.AddChunk(False, self.text_[self.chunk_start : ])
+    
+    return self.chunks_
+
+
+def HtmlToMessage(html, include_block_tags=False, description=''):
+  '''Takes a bit of HTML, which must contain only "inline" HTML elements,
+  and changes it into a tclib.Message.  This involves escaping any entities and
+  replacing any HTML code with placeholders.
+  
+  If include_block_tags is true, no error will be given if block tags (e.g.
+  <p> or <br>) are included in the HTML.
+  
+  Args:
+    html: 'Hello <b>[USERNAME]</b>, how&nbsp;<i>are</i> you?'
+    include_block_tags: False
+  
+  Return:
+    tclib.Message('Hello START_BOLD1USERNAMEEND_BOLD, '
+                  'howNBSPSTART_ITALICareEND_ITALIC you?',
+                  [ Placeholder('START_BOLD', '<b>', ''),
+                    Placeholder('USERNAME', '[USERNAME]', ''),
+                    Placeholder('END_BOLD', '</b>', ''),
+                    Placeholder('START_ITALIC', '<i>', ''),
+                    Placeholder('END_ITALIC', '</i>', ''), ])
+  '''
+  # Approach is:
+  # - first placeholderize, finding <elements>, [REPLACEABLES] and &nbsp;
+  # - then escape all character entities in text in-between placeholders
+  
+  parts = []  # List of strings (for text chunks) and tuples (ID, original)
+              # for placeholders
+  
+  count_names = {}  # Map of base names to number of times used
+  end_names = {}  # Map of base names to stack of end tags (for correct nesting)
+  
+  def MakeNameClosure(base, type = ''):
+    '''Returns a closure that can be called once all names have been allocated
+    to return the final name of the placeholder.  This allows us to minimally
+    number placeholders for non-overlap.
+    
+    Also ensures that END_XXX_Y placeholders have the same Y as the
+    corresponding BEGIN_XXX_Y placeholder when we have nested tags of the same
+    type.
+    
+    Args:
+      base: 'phname'
+      type: '' | 'begin' | 'end'
+    
+    Return:
+      Closure()
+    '''
+    name = base
+    if type != '':
+      name = ('%s_%s' % (type, base)).upper()
+
+    if name in count_names.keys():
+      count_names[name] += 1
+    else:
+      count_names[name] = 1
+    
+    def MakeFinalName(name_ = name, index = count_names[name] - 1):
+      if (type.lower() == 'end' and
+          base in end_names.keys() and len(end_names[base])):
+        return end_names[base].pop(-1)  # For correct nesting
+      if count_names[name_] != 1:
+        name_ = '%s_%s' % (name_, _SUFFIXES[index])
+        # We need to use a stack to ensure that the end-tag suffixes match
+        # the begin-tag suffixes.  Only needed when more than one tag of the
+        # same type.
+        if type == 'begin':
+          end_name = ('END_%s_%s' % (base, _SUFFIXES[index])).upper()
+          if base in end_names.keys():
+            end_names[base].append(end_name)
+          else:
+            end_names[base] = [end_name]
+      
+      return name_
+    
+    return MakeFinalName
+  
+  current = 0
+  
+  while current < len(html):
+    m = _NBSP.match(html[current:])
+    if m:
+      parts.append((MakeNameClosure('SPACE'), m.group()))
+      current += m.end()
+      continue
+    
+    m = _REPLACEABLE.match(html[current:])
+    if m:
+      # Replaceables allow - but placeholders don't, so replace - with _
+      ph_name = MakeNameClosure('X_%s_X' % m.group('name').replace('-', '_'))
+      parts.append((ph_name, m.group()))
+      current += m.end()
+      continue
+    
+    m = _SPECIAL_ELEMENT.match(html[current:])
+    if m:
+      if not include_block_tags:
+        raise exception.BlockTagInTranslateableChunk(html)
+      element_name = 'block'  # for simplification
+      # Get the appropriate group name
+      for group in m.groupdict().keys():
+        if m.groupdict()[group]:
+          break
+      parts.append((MakeNameClosure(element_name, 'begin'),
+                    html[current : current + m.start(group)]))
+      parts.append(m.group(group))
+      parts.append((MakeNameClosure(element_name, 'end'),
+                    html[current + m.end(group) : current + m.end()]))
+      current += m.end()
+      continue
+    
+    m = _ELEMENT.match(html[current:])
+    if m:
+      element_name = m.group('element').lower()
+      if not include_block_tags and not element_name in _INLINE_TAGS:
+        raise exception.BlockTagInTranslateableChunk(html[current:])
+      if element_name in _HTML_PLACEHOLDER_NAMES:  # use meaningful names
+        element_name = _HTML_PLACEHOLDER_NAMES[element_name]
+      
+      # Make a name for the placeholder
+      type = ''
+      if not m.group('empty'):
+        if m.group('closing'):
+          type = 'end'
+        else:
+          type = 'begin'
+      parts.append((MakeNameClosure(element_name, type), m.group()))
+      current += m.end()
+      continue
+    
+    if len(parts) and isinstance(parts[-1], types.StringTypes):
+      parts[-1] += html[current]
+    else:
+      parts.append(html[current])
+    current += 1
+  
+  msg_text = ''
+  placeholders = []
+  for part in parts:
+    if isinstance(part, types.TupleType):
+      final_name = part[0]()
+      original = part[1]
+      msg_text += final_name
+      placeholders.append(tclib.Placeholder(final_name, original, '(HTML code)'))
+    else:
+      msg_text += part
+  
+  msg = tclib.Message(text=msg_text, placeholders=placeholders,
+                      description=description)
+  content = msg.GetContent()
+  for ix in range(len(content)):
+    if isinstance(content[ix], types.StringTypes):
+      content[ix] = util.UnescapeHtml(content[ix], replace_nbsp=False)
+  
+  return msg
+
+
+class TrHtml(interface.GathererBase):
+  '''Represents a document or message in the template format used by
+  Total Recall for HTML documents.'''
+  
+  def __init__(self, text):
+    '''Creates a new object that represents 'text'.
+    Args:
+      text: '<html>...</html>'
+    '''
+    super(type(self), self).__init__()
+    
+    self.text_ = text      
+    self.have_parsed_ = False
+    self.skeleton_ = []  # list of strings and MessageClique objects
+  
+  def GetText(self):
+    '''Returns the original text of the HTML document'''
+    return self.text_
+  
+  def GetCliques(self):
+    '''Returns the message cliques for each translateable message in the
+    document.'''
+    return filter(lambda x: isinstance(x, clique.MessageClique), self.skeleton_)
+
+  def Translate(self, lang, pseudo_if_not_available=True,
+                skeleton_gatherer=None, fallback_to_english=False):
+    '''Returns this document with translateable messages filled with
+    the translation for language 'lang'.
+    
+    Args:
+      lang: 'en'
+      pseudo_if_not_available: True
+    
+    Return:
+      'ID_THIS_SECTION TYPE\n...BEGIN\n  "Translated message"\n......\nEND
+    
+    Raises:
+      grit.exception.NotReady() if used before Parse() has been successfully
+      called.
+      grit.exception.NoSuchTranslation() if 'pseudo_if_not_available' is false
+      and there is no translation for the requested language.
+    '''
+    if len(self.skeleton_) == 0:
+      raise exception.NotReady()
+    
+    # TODO(joi) Implement support for skeleton gatherers here.
+    
+    out = []
+    for item in self.skeleton_:
+      if isinstance(item, types.StringTypes):
+        out.append(item)
+      else:
+        msg = item.MessageForLanguage(lang,
+                                      pseudo_if_not_available,
+                                      fallback_to_english)
+        for content in msg.GetContent():
+          if isinstance(content, tclib.Placeholder):
+            out.append(content.GetOriginal())
+          else:
+            # We escape " characters to increase the chance that attributes
+            # will be properly escaped.
+            out.append(util.EscapeHtml(content, True))
+      
+    return ''.join(out)
+  
+  
+  # Parsing is done in two phases:  First, we break the document into
+  # translateable and nontranslateable chunks.  Second, we run through each
+  # translateable chunk and insert placeholders for any HTML elements, unescape
+  # escaped characters, etc.
+  def Parse(self):    
+    if self.have_parsed_:
+      return
+    self.have_parsed_ = True
+
+    text = self.text_
+    
+    # First handle the silly little [!]-prefixed header because it's not
+    # handled by our HTML parsers.
+    m = _SILLY_HEADER.match(text)
+    if m:
+      self.skeleton_.append(text[:m.start('title')])
+      self.skeleton_.append(self.uberclique.MakeClique(
+        tclib.Message(text=text[m.start('title'):m.end('title')])))
+      self.skeleton_.append(text[m.end('title') : m.end()])
+      text = text[m.end():]
+    
+    chunks = HtmlChunks().Parse(text)
+    
+    for chunk in chunks:
+      if chunk[0]:  # Chunk is translateable
+        self.skeleton_.append(self.uberclique.MakeClique(
+          HtmlToMessage(chunk[1], description=chunk[2])))
+      else:
+        self.skeleton_.append(chunk[1])
+  
+    # Go through the skeleton and change any messages that consist solely of
+    # placeholders and whitespace into nontranslateable strings.
+    for ix in range(len(self.skeleton_)):
+      got_text = False
+      if isinstance(self.skeleton_[ix], clique.MessageClique):
+        msg = self.skeleton_[ix].GetMessage()
+        for item in msg.GetContent():
+          if (isinstance(item, types.StringTypes) and _NON_WHITESPACE.search(item)
+              and item != '&nbsp;'):
+            got_text = True
+            break
+        if not got_text:
+          self.skeleton_[ix] = msg.GetRealContent()
+  
+  
+  # Static method
+  def FromFile(html, extkey=None, encoding = 'utf-8'):
+    '''Creates a TrHtml object from the contents of 'html' which are decoded
+    using 'encoding'.  Returns a new TrHtml object, upon which Parse() has not
+    been called.
+    
+    Args:
+      html: file('') | 'filename.html'
+      extkey: ignored
+      encoding: 'utf-8' (note that encoding is ignored if 'html' is not a file
+                         name but instead an open file or file-like object)
+    
+    Return:
+      TrHtml(text_of_file)
+    '''
+    if isinstance(html, types.StringTypes):
+      html = util.WrapInputStream(file(html, 'r'), encoding)
+    doc = html.read()
+
+    # Ignore the BOM character if the document starts with one.
+    if len(doc) and doc[0] == u'\ufeff':
+      doc = doc[1:]
+
+    return TrHtml(doc)
+  FromFile = staticmethod(FromFile)

tools/grit/grit/gather/tr_html_unittest.py

@@ -0,0 +1,437 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.gather.tr_html'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import types
+import unittest
+
+from grit.gather import tr_html
+from grit import clique
+from grit import util
+
+
+class ParserUnittest(unittest.TestCase):
+  def testChunking(self):
+    p = tr_html.HtmlChunks()
+    chunks = p.Parse('<p>Hello <b>dear</b> how <i>are</i>you?<p>Fine!')
+    self.failUnless(chunks == [
+      (False, '<p>', ''), (True, 'Hello <b>dear</b> how <i>are</i>you?', ''),
+      (False, '<p>', ''), (True, 'Fine!', '')])
+    
+    chunks = p.Parse('<p> Hello <b>dear</b> how <i>are</i>you? <p>Fine!')
+    self.failUnless(chunks == [
+      (False, '<p> ', ''), (True, 'Hello <b>dear</b> how <i>are</i>you?', ''),
+      (False, ' <p>', ''), (True, 'Fine!', '')])
+    
+    chunks = p.Parse('<p> Hello <b>dear how <i>are you? <p> Fine!')
+    self.failUnless(chunks == [
+      (False, '<p> ', ''), (True, 'Hello <b>dear how <i>are you?', ''),
+      (False, ' <p> ', ''), (True, 'Fine!', '')])
+    
+    # Ensure translateable sections that start with inline tags contain
+    # the starting inline tag.
+    chunks = p.Parse('<b>Hello!</b> how are you?<p><i>I am fine.</i>')
+    self.failUnless(chunks == [
+      (True, '<b>Hello!</b> how are you?', ''), (False, '<p>', ''),
+      (True, '<i>I am fine.</i>', '')])
+    
+    # Ensure translateable sections that end with inline tags contain
+    # the ending inline tag.
+    chunks = p.Parse("Hello! How are <b>you?</b><p><i>I'm fine!</i>")
+    self.failUnless(chunks == [
+      (True, 'Hello! How are <b>you?</b>', ''), (False, '<p>', ''),
+      (True, "<i>I'm fine!</i>", '')])
+
+    # Check capitals and explicit descriptions
+    chunks = p.Parse('<!-- desc=bingo! --><B>Hello!</B> how are you?<P><I>I am fine.</I>')
+    self.failUnless(chunks == [
+      (True, '<B>Hello!</B> how are you?', 'bingo!'), (False, '<P>', ''),
+      (True, '<I>I am fine.</I>', '')])
+    chunks = p.Parse('<B><!-- desc=bingo! -->Hello!</B> how are you?<P><I>I am fine.</I>')
+    self.failUnless(chunks == [
+      (True, '<B>Hello!</B> how are you?', 'bingo!'), (False, '<P>', ''),
+      (True, '<I>I am fine.</I>', '')])
+    # Linebreaks get changed to spaces just like any other HTML content
+    chunks = p.Parse('<B>Hello!</B> <!-- desc=bi\nngo\n! -->how are you?<P><I>I am fine.</I>')
+    self.failUnless(chunks == [
+      (True, '<B>Hello!</B> how are you?', 'bi ngo !'), (False, '<P>', ''),
+      (True, '<I>I am fine.</I>', '')])
+    
+    # In this case, because the explicit description appears after the first
+    # translateable, it will actually apply to the second translateable.
+    chunks = p.Parse('<B>Hello!</B> how are you?<!-- desc=bingo! --><P><I>I am fine.</I>')
+    self.failUnless(chunks == [
+      (True, '<B>Hello!</B> how are you?', ''), (False, '<P>', ''),
+      (True, '<I>I am fine.</I>', 'bingo!')])
+        
+    # Check that replaceables within block tags (where attributes would go) are
+    # handled correctly.
+    chunks = p.Parse('<b>Hello!</b> how are you?<p [BINGO] [$~BONGO~$]>'
+                     '<i>I am fine.</i>')
+    self.failUnless(chunks == [
+      (True, '<b>Hello!</b> how are you?', ''),
+      (False, '<p [BINGO] [$~BONGO~$]>', ''),
+      (True, '<i>I am fine.</i>', '')])
+    
+    # Check that the contents of preformatted tags preserve line breaks.
+    chunks = p.Parse('<textarea>Hello\nthere\nhow\nare\nyou?</textarea>')
+    self.failUnless(chunks == [(False, '<textarea>', ''),
+      (True, 'Hello\nthere\nhow\nare\nyou?', ''), (False, '</textarea>', '')])
+    
+    # ...and that other tags' line breaks are converted to spaces
+    chunks = p.Parse('<p>Hello\nthere\nhow\nare\nyou?</p>')
+    self.failUnless(chunks == [(False, '<p>', ''),
+      (True, 'Hello there how are you?', ''), (False, '</p>', '')])
+  
+  def testTranslateableAttributes(self):
+    p = tr_html.HtmlChunks()
+
+    # Check that the translateable attributes in <img>, <submit>, <button> and
+    # <text> elements buttons are handled correctly.
+    chunks = p.Parse('<img src=bingo.jpg alt="hello there">'
+                     '<input type=submit value="hello">'
+                     '<input type="button" value="hello">'
+                     '<input type=\'text\' value=\'Howdie\'>')
+    self.failUnless(chunks == [
+      (False, '<img src=bingo.jpg alt="', ''), (True, 'hello there', ''),
+      (False, '"><input type=submit value="', ''), (True, 'hello', ''),
+      (False, '"><input type="button" value="', ''), (True, 'hello', ''),
+      (False, '"><input type=\'text\' value=\'', ''), (True, 'Howdie', ''),
+      (False, '\'>', '')])
+  
+  
+  def testTranslateableHtmlToMessage(self):
+    msg = tr_html.HtmlToMessage(
+      'Hello <b>[USERNAME]</b>, &lt;how&gt;&nbsp;<i>are</i> you?')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    'Hello BEGIN_BOLDX_USERNAME_XEND_BOLD, '
+                    '<how>&nbsp;BEGIN_ITALICareEND_ITALIC you?')
+    
+    msg = tr_html.HtmlToMessage('<b>Hello</b><I>Hello</I><b>Hello</b>')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    'BEGIN_BOLD_1HelloEND_BOLD_1BEGIN_ITALICHelloEND_ITALIC'
+                    'BEGIN_BOLD_2HelloEND_BOLD_2')
+    
+    # Check that nesting (of the <font> tags) is handled correctly - i.e. that
+    # the closing placeholder numbers match the opening placeholders.
+    msg = tr_html.HtmlToMessage(
+      '''<font size=-1><font color=#FF0000>Update!</font> '''
+      '''<a href='http://desktop.google.com/whatsnew.html?hl=[$~LANG~$]'>'''
+      '''New Features</a>: Now search PDFs, MP3s, Firefox web history, and '''
+      '''more</font>''')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    'BEGIN_FONT_1BEGIN_FONT_2Update!END_FONT_2 BEGIN_LINK'
+                    'New FeaturesEND_LINK: Now search PDFs, MP3s, Firefox '
+                    'web history, and moreEND_FONT_1')
+    
+    msg = tr_html.HtmlToMessage('''<a href='[$~URL~$]'><b>[NUM][CAT]</b></a>''')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres == 'BEGIN_LINKBEGIN_BOLDX_NUM_XX_CAT_XEND_BOLDEND_LINK')
+    
+    msg = tr_html.HtmlToMessage(
+      '''<font size=-1><a class=q onClick='return window.qs?qs(this):1' '''
+      '''href='http://[WEBSERVER][SEARCH_URI]'>Desktop</a></font>&nbsp;&nbsp;'''
+      '''&nbsp;&nbsp;''')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    '''BEGIN_FONTBEGIN_LINKDesktopEND_LINKEND_FONTSPACE''')
+    
+    msg = tr_html.HtmlToMessage(
+      '''<br><br><center><font size=-2>&copy;2005 Google </font></center>''', 1)
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    u'BEGIN_BREAK_1BEGIN_BREAK_2BEGIN_CENTERBEGIN_FONT\xa92005'
+                    u' Google END_FONTEND_CENTER')
+    
+    msg = tr_html.HtmlToMessage(
+      '''&nbsp;-&nbsp;<a class=c href=[$~CACHE~$]>Cached</a>''')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    '&nbsp;-&nbsp;BEGIN_LINKCachedEND_LINK')
+    
+    # Check that upper-case tags are handled correctly.
+    msg = tr_html.HtmlToMessage(
+      '''You can read the <A HREF='http://desktop.google.com/privacypolicy.'''
+      '''html?hl=[LANG_CODE]'>Privacy Policy</A> and <A HREF='http://desktop'''
+      '''.google.com/privacyfaq.html?hl=[LANG_CODE]'>Privacy FAQ</A> online.''')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres ==
+                    'You can read the BEGIN_LINK_1Privacy PolicyEND_LINK_1 and '
+                    'BEGIN_LINK_2Privacy FAQEND_LINK_2 online.')
+    
+    # Check that tags with linebreaks immediately preceding them are handled
+    # correctly.
+    msg = tr_html.HtmlToMessage(
+      '''You can read the
+<A HREF='http://desktop.google.com/privacypolicy.html?hl=[LANG_CODE]'>Privacy Policy</A>
+and <A HREF='http://desktop.google.com/privacyfaq.html?hl=[LANG_CODE]'>Privacy FAQ</A> online.''')
+    pres = msg.GetPresentableContent()
+    self.failUnless(pres == '''You can read the
+BEGIN_LINK_1Privacy PolicyEND_LINK_1
+and BEGIN_LINK_2Privacy FAQEND_LINK_2 online.''')
+
+
+
+class TrHtmlUnittest(unittest.TestCase):
+  def testTable(self):
+    html = tr_html.TrHtml('''<table class="shaded-header"><tr>
+<td class="header-element b expand">Preferences</td>
+<td class="header-element s">
+<a href="http://desktop.google.com/preferences.html">Preferences&nbsp;Help</a>
+</td>
+</tr></table>''')
+    html.Parse()
+    self.failUnless(html.skeleton_[3].GetMessage().GetPresentableContent() ==
+                    'BEGIN_LINKPreferences&nbsp;HelpEND_LINK')
+  
+  def testSubmitAttribute(self):
+    html = tr_html.TrHtml('''</td>
+<td class="header-element"><input type=submit value="Save Preferences"
+name=submit2></td>
+</tr></table>''')
+    html.Parse()
+    self.failUnless(html.skeleton_[1].GetMessage().GetPresentableContent() ==
+                    'Save Preferences')
+  
+  def testWhitespaceAfterInlineTag(self):
+    '''Test that even if there is whitespace after an inline tag at the start
+    of a translateable section the inline tag will be included.
+    '''
+    html = tr_html.TrHtml('''<label for=DISPLAYNONE><font size=-1> Hello</font>''')
+    html.Parse()
+    self.failUnless(html.skeleton_[1].GetMessage().GetRealContent() ==
+                    '<font size=-1> Hello</font>')
+  
+  def testSillyHeader(self):
+    html = tr_html.TrHtml('''[!]
+title\tHello
+bingo
+bongo
+bla
+
+<p>Other stuff</p>''')
+    html.Parse()
+    content = html.skeleton_[1].GetMessage().GetRealContent()
+    self.failUnless(content == 'Hello')
+    self.failUnless(html.skeleton_[-1] == '</p>')
+    # Right after the translateable the nontranslateable should start with
+    # a linebreak (this catches a bug we had).
+    self.failUnless(html.skeleton_[2][0] == '\n')
+  
+  
+  def testExplicitDescriptions(self):
+    html = tr_html.TrHtml('Hello [USER]<br/><!-- desc=explicit --><input type="button">Go!</input>')
+    html.Parse()
+    msg = html.GetCliques()[1].GetMessage()
+    self.failUnless(msg.GetDescription() == 'explicit')
+    self.failUnless(msg.GetRealContent() == 'Go!')
+  
+  
+  def testRegressionInToolbarAbout(self):
+    html = tr_html.TrHtml.FromFile(
+      util.PathFromRoot(r'grit/test/data/toolbar_about.html'))
+    html.Parse()
+    cliques = html.GetCliques()
+    for cl in cliques:
+      content = cl.GetMessage().GetRealContent()
+      if content.count('De parvis grandis acervus erit'):
+        self.failIf(content.count('$/translate'))
+
+  
+  def HtmlFromFileWithManualCheck(self, f):
+    html = tr_html.TrHtml.FromFile(f)
+    html.Parse()
+    
+    # For manual results inspection only...
+    list = []
+    for item in html.skeleton_:
+      if isinstance(item, types.StringTypes):
+        list.append(item)
+      else:
+        list.append(item.GetMessage().GetPresentableContent())
+    
+    return html
+
+
+  def testPrivacyHtml(self):
+    html = self.HtmlFromFileWithManualCheck(
+      util.PathFromRoot(r'grit/test/data/privacy.html'))
+    
+    self.failUnless(html.skeleton_[1].GetMessage().GetRealContent() ==
+                    'Privacy and Google Desktop Search')
+    self.failUnless(html.skeleton_[3].startswith('<'))    
+    self.failUnless(len(html.skeleton_) > 10)
+
+
+  def testPreferencesHtml(self):
+    html = self.HtmlFromFileWithManualCheck(
+      util.PathFromRoot(r'grit/test/data/preferences.html'))
+    
+    # Verify that we don't get '[STATUS-MESSAGE]' as the original content of
+    # one of the MessageClique objects (it would be a placeholder-only message
+    # and we're supposed to have stripped those).
+    
+    for item in filter(lambda x: isinstance(x, clique.MessageClique),
+                       html.skeleton_):
+      if (item.GetMessage().GetRealContent() == '[STATUS-MESSAGE]' or
+          item.GetMessage().GetRealContent() == '[ADDIN-DO] [ADDIN-OPTIONS]'):
+        self.fail()
+    
+    self.failUnless(len(html.skeleton_) > 100)
+  
+  def AssertNumberOfTranslateables(self, files, num):
+    '''Fails if any of the files in files don't have exactly
+    num translateable sections.
+    
+    Args:
+      files: ['file1', 'file2']
+      num: 3
+    '''
+    for f in files:
+      f = util.PathFromRoot(r'grit/test/data/%s' % f)
+      html = self.HtmlFromFileWithManualCheck(f)
+      self.failUnless(len(html.GetCliques()) == num)
+  
+  def testFewTranslateables(self):
+    self.AssertNumberOfTranslateables(['browser.html', 'email_thread.html',
+                                       'header.html', 'mini.html',
+                                       'oneclick.html', 'script.html',
+                                       'time_related.html', 'versions.html'], 0)
+    self.AssertNumberOfTranslateables(['footer.html', 'hover.html'], 1)
+  
+  def testOtherHtmlFilesForManualInspection(self):
+    files = [
+      'about.html', 'bad_browser.html', 'cache_prefix.html',
+      'cache_prefix_file.html', 'chat_result.html', 'del_footer.html',
+      'del_header.html', 'deleted.html', 'details.html', 'email_result.html',
+      'error.html', 'explicit_web.html', 'footer.html',
+      'homepage.html', 'indexing_speed.html',
+      'install_prefs.html', 'install_prefs2.html',
+      'oem_enable.html', 'oem_non_admin.html', 'onebox.html',
+      'password.html', 'quit_apps.html', 'recrawl.html',
+      'searchbox.html', 'sidebar_h.html', 'sidebar_v.html', 'status.html',
+    ]
+    for f in files:
+      self.HtmlFromFileWithManualCheck(
+        util.PathFromRoot(r'grit/test/data/%s' % f))
+  
+  def testTranslate(self):
+    # Note that the English translation of documents that use character
+    # literals (e.g. &copy;) will not be the same as the original document
+    # because the character literal will be transformed into the Unicode
+    # character itself.  So for this test we choose some relatively complex
+    # HTML without character entities (but with &nbsp; because that's handled
+    # specially).
+    html = tr_html.TrHtml('''  <script>
+      <!--
+      function checkOffice() { var w = document.getElementById("h7");
+      var e = document.getElementById("h8"); var o = document.getElementById("h10");
+      if (!(w.checked || e.checked)) { o.checked=0;o.disabled=1;} else {o.disabled=0;} }
+      // -->
+        </script>
+        <input type=checkbox [CHECK-DOC] name=DOC id=h7 onclick='checkOffice()'>
+        <label for=h7> Word</label><br>
+        <input type=checkbox [CHECK-XLS] name=XLS id=h8 onclick='checkOffice()'>
+        <label for=h8> Excel</label><br>
+        <input type=checkbox [CHECK-PPT] name=PPT id=h9>
+        <label for=h9> PowerPoint</label><br>
+        </span></td><td nowrap valign=top><span class="s">
+        <input type=checkbox [CHECK-PDF] name=PDF id=hpdf>
+        <label for=hpdf> PDF</label><br>
+        <input type=checkbox [CHECK-TXT] name=TXT id=h6>
+        <label for=h6> Text, media, and other files</label><br>
+       </tr>&nbsp;&nbsp;
+       <tr><td nowrap valign=top colspan=3><span class="s"><br />
+       <input type=checkbox [CHECK-SECUREOFFICE] name=SECUREOFFICE id=h10>
+        <label for=h10> Password-protected Office documents (Word, Excel)</label><br />
+        <input type=checkbox [DISABLED-HTTPS] [CHECK-HTTPS] name=HTTPS id=h12><label
+        for=h12> Secure pages (HTTPS) in web history</label></span></td></tr>
+      </table>''')
+    html.Parse()
+    trans = html.Translate('en')
+    if (html.GetText() != trans):
+      self.fail()
+  
+  
+  def testHtmlToMessageWithBlockTags(self):
+    msg = tr_html.HtmlToMessage(
+      'Hello<p>Howdie<img alt="bingo" src="image.gif">', True)
+    result = msg.GetPresentableContent()
+    self.failUnless(
+      result == 'HelloBEGIN_PARAGRAPHHowdieBEGIN_BLOCKbingoEND_BLOCK')
+    
+    msg = tr_html.HtmlToMessage(
+      'Hello<p>Howdie<input type="button" value="bingo">', True)
+    result = msg.GetPresentableContent()
+    self.failUnless(
+      result == 'HelloBEGIN_PARAGRAPHHowdieBEGIN_BLOCKbingoEND_BLOCK')
+  
+  
+  def testHtmlToMessageRegressions(self):
+    msg = tr_html.HtmlToMessage(' - ', True)
+    result = msg.GetPresentableContent()
+    self.failUnless(result == ' - ')
+    
+
+  def testEscapeUnescaped(self):
+    text = '&copy;&nbsp; & &quot;&lt;hello&gt;&quot;'
+    unescaped = util.UnescapeHtml(text)
+    self.failUnless(unescaped == u'\u00a9\u00a0 & "<hello>"')
+    escaped_unescaped = util.EscapeHtml(unescaped, True)
+    self.failUnless(escaped_unescaped ==
+                    u'\u00a9\u00a0 &amp; &quot;&lt;hello&gt;&quot;')
+
+  def testRegressionCjkHtmlFile(self):
+    # TODO(joi) Fix this problem where unquoted attributes that
+    # have a value that is CJK characters causes the regular expression
+    # match never to return.  (culprit is the _ELEMENT regexp(
+    if False:
+      html = self.HtmlFromFileWithManualCheck(util.PathFromRoot(
+        r'grit/test/data/ko_oem_enable_bug.html'))
+      self.failUnless(True)
+  
+  def testRegressionCpuHang(self):
+    # If this regression occurs, the unit test will never return
+    html = tr_html.TrHtml(
+      '''<input type=text size=12 id=advFileTypeEntry [~SHOW-FILETYPE-BOX~] value="[EXT]" name=ext>''')
+    html.Parse()
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/gather/txt.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Supports making amessage from a text file.
+'''
+
+import types
+
+from grit.gather import interface
+from grit import tclib
+from grit import util
+
+
+class TxtFile(interface.GathererBase):
+  '''A text file gatherer.  Very simple, all text from the file becomes a
+  single clique.
+  '''
+  
+  def __init__(self, contents):
+    super(type(self), self).__init__()
+    self.text_ = contents
+    self.clique_ = None
+
+  def Parse(self):
+    self.clique_ = self.uberclique.MakeClique(tclib.Message(text=self.text_))
+    pass
+  
+  def GetText(self):
+    '''Returns the text of what is being gathered.'''
+    return self.text_
+  
+  def GetTextualIds(self):
+    return []
+    
+  def GetCliques(self):
+    '''Returns the MessageClique objects for all translateable portions.'''
+    return [self.clique_]
+  
+  def Translate(self, lang, pseudo_if_not_available=True,
+                skeleton_gatherer=None, fallback_to_english=False):
+    return self.clique_.MessageForLanguage(lang,
+                                           pseudo_if_not_available,
+                                           fallback_to_english).GetRealContent()
+  
+  def FromFile(filename_or_stream, extkey=None, encoding = 'cp1252'):
+    if isinstance(filename_or_stream, types.StringTypes):
+      filename_or_stream = util.WrapInputStream(file(filename_or_stream, 'rb'), encoding)
+    return TxtFile(filename_or_stream.read())
+  FromFile = staticmethod(FromFile)

tools/grit/grit/gather/txt_unittest.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for TxtFile gatherer'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+
+import StringIO
+import unittest
+
+from grit.gather import txt
+
+
+class TxtUnittest(unittest.TestCase):
+  def testGather(self):
+    input = StringIO.StringIO('Hello there\nHow are you?')
+    gatherer = txt.TxtFile.FromFile(input)
+    gatherer.Parse()
+    self.failUnless(gatherer.GetText() == input.getvalue())
+    self.failUnless(len(gatherer.GetCliques()) == 1)
+    self.failUnless(gatherer.GetCliques()[0].GetMessage().GetRealContent() ==
+                    input.getvalue())
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/grd_reader.py

@@ -0,0 +1,166 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Class for reading GRD files into memory, without processing them.
+'''
+
+import os.path
+import types
+import xml.sax
+import xml.sax.handler
+
+from grit import exception
+from grit.node import base
+from grit.node import mapping
+from grit import util
+
+
+class StopParsingException(Exception):
+  '''An exception used to stop parsing.'''
+  pass
+
+
+class GrdContentHandler(xml.sax.handler.ContentHandler):
+  def __init__(self, stop_after=None, debug=False):
+    # Invariant of data:
+    # 'root' is the root of the parse tree being created, or None if we haven't
+    # parsed out any elements.
+    # 'stack' is the a stack of elements that we push new nodes onto and
+    # pop from when they finish parsing, or [] if we are not currently parsing.
+    # 'stack[-1]' is the top of the stack.
+    self.root = None
+    self.stack = []
+    self.stop_after = stop_after
+    self.debug = debug
+  
+  def startElement(self, name, attrs):
+    assert not self.root or len(self.stack) > 0
+    
+    if self.debug:
+      attr_list = []
+      for attr in attrs.getNames():
+        attr_list.append('%s="%s"' % (attr, attrs.getValue(attr)))
+      if len(attr_list) == 0: attr_list = ['(none)']
+      attr_list = ' '.join(attr_list)
+      print "Starting parsing of element %s with attributes %r" % (name, attr_list)
+    
+    typeattr = None
+    if 'type' in attrs.getNames():
+      typeattr = attrs.getValue('type')
+    
+    node = mapping.ElementToClass(name, typeattr)()
+    
+    if not self.root:
+      self.root = node
+      
+    if len(self.stack) > 0:
+      self.stack[-1].AddChild(node)
+      node.StartParsing(name, self.stack[-1])
+    else:
+      node.StartParsing(name, None)
+    
+    # Push
+    self.stack.append(node)
+    
+    for attr in attrs.getNames():
+      node.HandleAttribute(attr, attrs.getValue(attr))
+    
+  def endElement(self, name):
+    if self.debug:
+      print "End parsing of element %s" % name
+    # Pop
+    self.stack[-1].EndParsing()
+    assert len(self.stack) > 0
+    self.stack = self.stack[:-1]
+    if self.stop_after and name == self.stop_after:
+      raise StopParsingException()
+  
+  def characters(self, content):
+    if self.stack[-1]:
+      self.stack[-1].AppendContent(content)
+  
+  def ignorableWhitespace(self, whitespace):
+    # TODO(joi)  This is not supported by expat.  Should use a different XML parser?
+    pass
+
+
+def Parse(filename_or_stream, dir = None, flexible_root = False,
+          stop_after=None, debug=False):
+  '''Parses a GRD file into a tree of nodes (from grit.node).
+  
+  If flexible_root is False, the root node must be a <grit> element.  Otherwise
+  it can be any element.  The "own" directory of the file will only be fixed up
+  if the root node is a <grit> element.
+  
+  'dir' should point to the directory of the input file, or be the full path
+  to the input file (the filename will be stripped).
+  
+  If 'stop_after' is provided, the parsing will stop once the first node
+  with this name has been fully parsed (including all its contents).
+  
+  If 'debug' is true, lots of information about the parsing events will be
+  printed out during parsing of the file.
+  
+  Args:
+    filename_or_stream: './bla.xml'  (must be filename if dir is None)
+    dir: '.' or None (only if filename_or_stream is a filename)
+    flexible_root: True | False
+    stop_after: 'inputs'
+    debug: False
+  
+  Return:
+    Subclass of grit.node.base.Node
+    
+  Throws:
+    grit.exception.Parsing
+  '''  
+  handler = GrdContentHandler(stop_after=stop_after, debug=debug)
+  try:
+    xml.sax.parse(filename_or_stream, handler)
+  except StopParsingException:
+    assert stop_after
+    pass
+  except:
+    raise
+
+  if not flexible_root or hasattr(handler.root, 'SetOwnDir'):
+    assert isinstance(filename_or_stream, types.StringType) or dir != None
+    if not dir:
+      dir = util.dirname(filename_or_stream)
+      if len(dir) == 0:
+        dir = '.'
+    # Fix up the base_dir so it is relative to the input file.
+    handler.root.SetOwnDir(dir)
+  return handler.root
+
+
+if __name__ == '__main__':
+  util.ChangeStdoutEncoding()
+  print unicode(Parse(sys.argv[1]))

tools/grit/grit/grd_reader_unittest.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grd_reader package'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+import StringIO
+
+from grit import grd_reader
+from grit import constants
+from grit import util
+
+
+class GrdReaderUnittest(unittest.TestCase):
+  def testParsingAndXmlOutput(self):
+    input = u'''<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+  <release seq="3">
+    <includes>
+      <include name="ID_LOGO" file="images/logo.gif" type="gif" />
+    </includes>
+    <messages>
+      <if expr="True">
+        <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+          Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+        </message>
+      </if>
+    </messages>
+    <structures>
+      <structure name="IDD_NARROW_DIALOG" file="rc_files/dialogs.rc" type="dialog">
+        <skeleton variant_of_revision="3" expr="lang == 'fr-FR'" file="bla.rc" />
+      </structure>
+      <structure name="VS_VERSION_INFO" file="rc_files/version.rc" type="version" />
+    </structures>
+  </release>
+  <translations>
+    <file lang="nl" path="nl_translations.xtb" />
+  </translations>
+  <outputs>
+    <output type="rc_header" filename="resource.h" />
+    <output lang="en-US" type="rc_all" filename="resource.rc" />
+  </outputs>
+</grit>'''
+    pseudo_file = StringIO.StringIO(input)
+    tree = grd_reader.Parse(pseudo_file, '.')
+    output = unicode(tree)
+    # All but first two lines are the same (sans enc_check)
+    self.failUnless('\n'.join(input.split('\n')[2:]) ==
+                    '\n'.join(output.split('\n')[2:]))
+    self.failUnless(tree.GetNodeById('IDS_GREETING'))
+
+
+  def testStopAfter(self):
+    input = u'''<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+  <outputs>
+    <output filename="resource.h" type="rc_header" />
+    <output filename="resource.rc" lang="en-US" type="rc_all" />
+  </outputs>
+  <release seq="3">
+    <includes>
+      <include type="gif" name="ID_LOGO" file="images/logo.gif"/>
+    </includes>
+  </release>
+</grit>'''
+    pseudo_file = util.WrapInputStream(StringIO.StringIO(input))
+    tree = grd_reader.Parse(pseudo_file, '.', stop_after='outputs')
+    # only an <outputs> child
+    self.failUnless(len(tree.children) == 1)
+    self.failUnless(tree.children[0].name == 'outputs')
+  
+  def testLongLinesWithComments(self):
+    input = u'''<?xml version="1.0" encoding="UTF-8"?>
+<grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+  <release seq="3">
+    <messages>
+      <message name="IDS_GREETING" desc="Printed to greet the currently logged in user">
+        This is a very long line with no linebreaks yes yes it stretches on <!--
+        -->and on <!--
+        -->and on!
+      </message>
+    </messages>
+  </release>
+</grit>'''
+    pseudo_file = StringIO.StringIO(input)
+    tree = grd_reader.Parse(pseudo_file, '.')
+    
+    greeting = tree.GetNodeById('IDS_GREETING')
+    self.failUnless(greeting.GetCliques()[0].GetMessage().GetRealContent() ==
+                    'This is a very long line with no linebreaks yes yes it '
+                    'stretches on and on and on!')
+
+if __name__ == '__main__':
+  unittest.main()
+  

tools/grit/grit/grit-todo.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="windows-1252"?>
+<TODOLIST FILEFORMAT="6" PROJECTNAME="GRIT" NEXTUNIQUEID="56" FILEVERSION="69" LASTMODIFIED="2005-08-19">
+    <TASK STARTDATESTRING="2005-04-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38453.49975694" TITLE="check 'name' attribute is unique" TIMEESTUNITS="H" ID="2" PERCENTDONE="100" STARTDATE="38450.00000000" DONEDATESTRING="2005-04-11" POS="22" DONEDATE="38453.00000000"/>
+    <TASK STARTDATESTRING="2005-04-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38488.48189815" TITLE="import id-calculating code" TIMEESTUNITS="H" ID="3" PERCENTDONE="100" STARTDATE="38450.00000000" DONEDATESTRING="2005-05-16" POS="13" DONEDATE="38488.00000000"/>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38488.48209491" TITLE="Import tool for existing translations" TIMEESTUNITS="H" ID="6" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="12" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00805556" TITLE="Export XMBs" TIMEESTUNITS="H" ID="8" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-08" POS="20" DONEDATE="38511.00000000"/>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00924769" TITLE="Initial Integration" TIMEESTUNITS="H" ID="10" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-08" POS="10" DONEDATE="38511.00000000">
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38496.54048611" TITLE="parser for %s strings" TIMEESTUNITS="H" ID="4" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-24" POS="2" DONEDATE="38496.00000000"/>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38497.00261574" TITLE="import tool for existing RC files" TIMEESTUNITS="H" ID="5" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-25" POS="4" DONEDATE="38497.00000000">
+            <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38496.92990741" TITLE="handle button value= and img alt= in message HTML text" TIMEESTUNITS="H" ID="22" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-05-24" POS="1" DONEDATE="38496.00000000"/>
+            <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38497.00258102" TITLE="&amp;nbsp; bug" TIMEESTUNITS="H" ID="23" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-05-25" POS="2" DONEDATE="38497.00000000"/>
+        </TASK>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61171296" TITLE="grit build" TIMEESTUNITS="H" ID="7" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="6" DONEDATE="38490.00000000">
+            <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61168981" TITLE="use IDs gathered from gatherers for .h file" TIMEESTUNITS="H" ID="20" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="1" DONEDATE="38490.00000000"/>
+        </TASK>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.55199074" TITLE="SCons Integration" TIMEESTUNITS="H" ID="9" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-01" POS="1" DONEDATE="38504.00000000"/>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38490.61181713" TITLE="handle includes" TIMEESTUNITS="H" ID="12" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-05-18" POS="5" DONEDATE="38490.00000000"/>
+        <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38507.98567130" TITLE="output translated HTML templates" TIMEESTUNITS="H" ID="25" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-04" POS="3" DONEDATE="38507.00000000"/>
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38507.99394676" TITLE="bug: re-escape too much in RC dialogs etc." TIMEESTUNITS="H" ID="38" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-04" POS="7" DONEDATE="38507.00000000"/>
+    </TASK>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46444444" TITLE="handle structure variants" TIMEESTUNITS="H" ID="11" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="15" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46456019" TITLE="handle include variants" TIMEESTUNITS="H" ID="13" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="17" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46537037" TITLE="handle translateable text for includes (e.g. image text)" TIMEESTUNITS="H" ID="14" PERCENTDONE="100" STARTDATE="38488.00000000" DONEDATESTRING="2005-06-16" POS="14" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46712963" TITLE="ddoc" TIMEESTUNITS="H" ID="15" STARTDATE="38488.00000000" POS="4">
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46718750" TITLE="review comments miket" TIMEESTUNITS="H" ID="16" STARTDATE="38488.00000000" POS="2"/>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46722222" TITLE="review comments pdoyle" TIMEESTUNITS="H" ID="17" STARTDATE="38488.00000000" POS="1"/>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.46732639" TITLE="remove 'extkey' from structure" TIMEESTUNITS="H" ID="18" STARTDATE="38488.00000000" POS="3"/>
+        <TASK STARTDATESTRING="2005-05-16" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38489.53537037" TITLE="add 'encoding' to structure" TIMEESTUNITS="H" ID="19" STARTDATE="38488.00000000" POS="6"/>
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38503.55304398" TITLE="document limitation: emitter doesn't emit the translated HTML templates" TIMEESTUNITS="H" ID="30" STARTDATE="38503.00000000" POS="4"/>
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.58541667" TITLE="add 'internal_comment' to &lt;message&gt;" TIMEESTUNITS="H" ID="32" STARTDATE="38503.00000000" POS="5"/>
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.73391204" TITLE="&lt;outputs&gt; can not have paths (because of SCons integration - goes to build dir)" TIMEESTUNITS="H" ID="36" STARTDATE="38503.00000000" POS="9"/>
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38506.64265046" TITLE="&lt;identifers&gt; and &lt;identifier&gt; nodes" TIMEESTUNITS="H" ID="37" STARTDATE="38503.00000000" POS="10"/>
+        <TASK STARTDATESTRING="2005-06-23" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38526.62344907" TITLE="&lt;structure&gt; can have 'exclude_from_rc' attribute (default false)" TIMEESTUNITS="H" ID="47" STARTDATE="38526.00000000" POS="8"/>
+        <TASK STARTDATESTRING="2005-06-23" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38531.94135417" TITLE="add 'enc_check' to &lt;grit&gt;" TIMEESTUNITS="H" ID="48" STARTDATE="38526.00000000" POS="7"/>
+    </TASK>
+    <TASK STARTDATESTRING="2005-05-18" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38492.51549769" TITLE="handle nontranslateable messages (in MessageClique?)" TIMEESTUNITS="H" ID="21" PERCENTDONE="100" STARTDATE="38490.00000000" DONEDATESTRING="2005-06-16" POS="16" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70454861" TITLE="ask cprince about SCons builder in new mk system" TIMEESTUNITS="H" ID="24" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-02" POS="25" DONEDATE="38505.00000000"/>
+    <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38504.57436343" TITLE="fix AOL resource in trunk (&quot;???????&quot;)" TIMEESTUNITS="H" ID="26" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-01" POS="19" DONEDATE="38504.00000000"/>
+    <TASK STARTDATESTRING="2005-05-24" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38498.53893519" TITLE="rc_all vs. rc_translateable vs. rc_nontranslateable" TIMEESTUNITS="H" ID="27" PERCENTDONE="100" STARTDATE="38496.00000000" DONEDATESTRING="2005-06-16" POS="6" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38509.45532407" TITLE="make separate .grb &quot;outputs&quot; file (and change SCons integ)  (??)" TIMEESTUNITS="H" ID="28" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-06" POS="8" DONEDATE="38509.00000000"/>
+    <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00939815" TITLE="fix unit tests so they run from any directory" TIMEESTUNITS="H" ID="33" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-08" POS="18" DONEDATE="38511.00000000"/>
+    <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38508.96640046" TITLE="Change R4 tool to CC correct team(s) on GRIT changes" TIMEESTUNITS="H" ID="39" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-05" POS="23" DONEDATE="38508.00000000"/>
+    <TASK STARTDATESTRING="2005-06-07" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00881944" TITLE="Document why wrapper.rc" TIMEESTUNITS="H" ID="40" PERCENTDONE="100" STARTDATE="38510.00000000" DONEDATESTRING="2005-06-08" POS="21" DONEDATE="38511.00000000"/>
+    <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00804398" TITLE="import XTBs" TIMEESTUNITS="H" ID="41" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="11" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00875000" TITLE="Nightly build integration" TIMEESTUNITS="H" ID="42" STARTDATE="38511.00000000" POS="3"/>
+    <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00891204" TITLE="BUGS" TIMEESTUNITS="H" ID="43" STARTDATE="38511.00000000" POS="24">
+        <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38513.03375000" TITLE="Should report error if RC-section structure refers to does not exist" TIMEESTUNITS="H" ID="44" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-10" POS="1" DONEDATE="38513.00000000"/>
+    </TASK>
+    <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.00981481" TITLE="NEW FEATURES" TIMEESTUNITS="H" ID="45" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="7" DONEDATE="38519.00000000">
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70077546" TITLE="Implement line-continuation feature (\ at end of line?)" TIMEESTUNITS="H" ID="34" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-16" POS="1" DONEDATE="38519.00000000"/>
+        <TASK STARTDATESTRING="2005-05-31" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38505.70262731" TITLE="Implement conditional inclusion &amp; reflect the conditionals from R3 RC file" TIMEESTUNITS="H" ID="35" PERCENTDONE="100" STARTDATE="38503.00000000" DONEDATESTRING="2005-06-16" POS="2" DONEDATE="38519.00000000"/>
+    </TASK>
+    <TASK STARTDATESTRING="2005-06-08" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38511.01046296" TITLE="TC integration (one-way TO the TC)" TIMEESTUNITS="H" ID="46" PERCENTDONE="100" STARTDATE="38511.00000000" DONEDATESTRING="2005-06-16" POS="5" DONEDATE="38519.00000000"/>
+    <TASK STARTDATESTRING="2005-06-30" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38533.59072917" TITLE="bazaar20 ad for GRIT help" TIMEESTUNITS="H" ID="49" STARTDATE="38533.00000000" POS="2">
+        <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72346065" TITLE="bazaar20 ideas" TIMEESTUNITS="H" ID="51" STARTDATE="38583.00000000" POS="1">
+            <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72354167" TITLE="GUI for adding/editing messages" TIMEESTUNITS="H" ID="52" STARTDATE="38583.00000000" POS="2"/>
+            <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.72365741" TITLE="XLIFF import/export" TIMEESTUNITS="H" ID="54" STARTDATE="38583.00000000" POS="1"/>
+        </TASK>
+    </TASK>
+    <TASK STARTDATESTRING="2005-06-30" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.73721065" TITLE="internal_comment for all resource nodes (not just &lt;message&gt;)" TIMEESTUNITS="H" ID="50" PERCENTDONE="100" STARTDATE="38533.00000000" DONEDATESTRING="2005-08-19" POS="9" DONEDATE="38583.73721065"/>
+    <TASK STARTDATESTRING="2005-08-19" PRIORITY="5" TIMEESPENTUNITS="H" LASTMOD="38583.73743056" TITLE="Preserve XML comments - this gives us line continuation and more" TIMEESTUNITS="H" ID="55" STARTDATE="38583.72326389" POS="1"/>
+</TODOLIST>

tools/grit/grit/grit_runner.py

@@ -0,0 +1,228 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Command processor for GRIT.  This is the script you invoke to run the various
+GRIT tools.
+'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import getopt
+
+from grit import util
+
+import grit.exception
+
+import grit.tool.build
+import grit.tool.count
+import grit.tool.diff_structures
+import grit.tool.menu_from_parts
+import grit.tool.newgrd
+import grit.tool.resize
+import grit.tool.rc2grd
+import grit.tool.test
+import grit.tool.transl2tc
+import grit.tool.unit
+
+
+# Copyright notice
+_COPYRIGHT = '''
+GRIT - the Google Resource and Internationalization Tool
+Copyright (c) Google Inc. %d
+''' % util.GetCurrentYear()
+
+# Keys for the following map
+_CLASS = 1
+_REQUIRES_INPUT = 2
+_HIDDEN = 3  # optional key - presence indicates tool is hidden
+
+
+# Maps tool names to the tool's module.  Done as a list of (key, value) tuples
+# instead of a map to preserve ordering.
+_TOOLS = [
+  ['build', { _CLASS : grit.tool.build.RcBuilder, _REQUIRES_INPUT : True }],
+  ['newgrd', { _CLASS  : grit.tool.newgrd.NewGrd, _REQUIRES_INPUT : False }],
+  ['rc2grd', { _CLASS : grit.tool.rc2grd.Rc2Grd, _REQUIRES_INPUT : False }],
+  ['transl2tc', { _CLASS : grit.tool.transl2tc.TranslationToTc,
+                 _REQUIRES_INPUT : False }],
+  ['sdiff', { _CLASS : grit.tool.diff_structures.DiffStructures,
+                       _REQUIRES_INPUT : False }],
+  ['resize', { _CLASS : grit.tool.resize.ResizeDialog, _REQUIRES_INPUT : True }],
+  ['unit', { _CLASS : grit.tool.unit.UnitTestTool, _REQUIRES_INPUT : False }],
+  ['count', { _CLASS : grit.tool.count.CountMessage, _REQUIRES_INPUT : True }],
+  ['test', { _CLASS: grit.tool.test.TestTool, _REQUIRES_INPUT : True, _HIDDEN : True }],
+  ['menufromparts', { _CLASS: grit.tool.menu_from_parts.MenuTranslationsFromParts,
+                     _REQUIRES_INPUT : True, _HIDDEN : True }],
+]
+
+
+def PrintUsage():
+  tool_list = ''
+  for (tool, info) in _TOOLS:
+    if not _HIDDEN in info.keys():
+      tool_list += '    %-12s %s\n' % (tool, info[_CLASS]().ShortDescription())
+  
+  # TODO(joi) Put these back into the usage when appropriate:
+  #
+  #  -d    Work disconnected.  This causes GRIT not to attempt connections with
+  #        e.g. Perforce.
+  #
+  #  -c    Use the specified Perforce CLIENT when talking to Perforce.
+  print '''Usage: grit [GLOBALOPTIONS] TOOL [args to tool]
+
+Global options:
+
+  -i INPUT  Specifies the INPUT file to use (a .grd file).  If this is not
+            specified, GRIT will look for the environment variable GRIT_INPUT.
+            If it is not present either, GRIT will try to find an input file
+            named 'resource.grd' in the current working directory.
+
+  -v        Print more verbose runtime information.
+  
+  -x        Print extremely verbose runtime information.  Implies -v
+  
+  -p FNAME  Specifies that GRIT should profile its execution and output the
+            results to the file FNAME.
+ 
+Tools:
+
+  TOOL can be one of the following:
+%s
+  For more information on how to use a particular tool, and the specific
+  arguments you can send to that tool, execute 'grit help TOOL'
+''' % (tool_list)
+
+
+class Options(object):
+  '''Option storage and parsing.'''
+  
+  def __init__(self):
+    self.disconnected = False
+    self.client = ''
+    self.input = None
+    self.verbose = False
+    self.extra_verbose = False
+    self.output_stream = sys.stdout
+    self.profile_dest = None
+  
+  def ReadOptions(self, args):
+    '''Reads options from the start of args and returns the remainder.'''
+    (opts, args) = getopt.getopt(args, 'g:dvxc:i:p:')
+    for (key, val) in opts:
+      if key == '-d': self.disconnected = True
+      elif key == '-c': self.client = val
+      elif key == '-i': self.input = val
+      elif key == '-v':
+        self.verbose = True
+        util.verbose = True
+      elif key == '-x':
+        self.verbose = True
+        util.verbose = True
+        self.extra_verbose = True
+        util.extra_verbose = True
+      elif key == '-p': self.profile_dest = val
+    
+    if not self.input:
+      if 'GRIT_INPUT' in os.environ:
+        self.input = os.environ['GRIT_INPUT']
+      else:
+        self.input = 'resource.grd'
+    
+    return args
+    
+  def __repr__(self):
+    return '(disconnected: %d, verbose: %d, client: %s, input: %s)' % (
+      self.disconnected, self.verbose, self.client, self.input)
+
+
+def _GetToolInfo(tool):
+  '''Returns the info map for the tool named 'tool' or None if there is no
+  such tool.'''
+  matches = filter(lambda t: t[0] == tool, _TOOLS)
+  if not len(matches):
+    return None
+  else:
+    return matches[0][1]
+
+
+def Main(args):
+  '''Parses arguments and does the appropriate thing.'''
+  util.ChangeStdoutEncoding()
+  print _COPYRIGHT
+  
+  if not len(args) or len(args) == 1 and args[0] == 'help':
+    PrintUsage()
+    return 0
+  elif len(args) == 2 and args[0] == 'help':
+    tool = args[1].lower()
+    if not _GetToolInfo(tool):
+      print "No such tool.  Try running 'grit help' for a list of tools."
+      return 2
+    
+    print ("Help for 'grit %s' (for general help, run 'grit help'):\n"
+           % (tool))
+    print _GetToolInfo(tool)[_CLASS].__doc__
+    return 0
+  else:
+    options = Options()
+    args = options.ReadOptions(args)  # args may be shorter after this
+    tool = args[0]
+    if not _GetToolInfo(tool):
+      print "No such tool.  Try running 'grit help' for a list of tools."
+      return 2
+
+    try:
+      if _GetToolInfo(tool)[_REQUIRES_INPUT]:
+        os.stat(options.input)
+    except OSError:
+      print ('Input file %s not found.\n'
+             'To specify a different input file:\n'
+             '  1. Use the GRIT_INPUT environment variable.\n'
+             '  2. Use the -i command-line option.  This overrides '
+             'GRIT_INPUT.\n'
+             '  3. Specify neither GRIT_INPUT or -i and GRIT will try to load '
+             "'resource.grd'\n"
+             '     from the current directory.' % options.input)
+      return 2
+
+    toolobject = _GetToolInfo(tool)[_CLASS]()
+    if options.profile_dest:
+      import hotshot
+      prof = hotshot.Profile(options.profile_dest)
+      prof.runcall(toolobject.Run, options, args[1:])
+    else:
+      toolobject.Run(options, args[1:])
+
+
+if __name__ == '__main__':
+  sys.exit(Main(sys.argv[1:]))

tools/grit/grit/grit_runner_unittest.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.py'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+import StringIO
+
+from grit import util
+import grit.grit_runner
+
+class OptionArgsUnittest(unittest.TestCase):
+  def setUp(self):
+    self.buf = StringIO.StringIO()
+    self.old_stdout = sys.stdout
+    sys.stdout = self.buf
+  
+  def tearDown(self):
+    sys.stdout = self.old_stdout
+    
+  def testSimple(self):
+    grit.grit_runner.Main(['-i',
+                           util.PathFromRoot('grit/test/data/simple-input.xml'),
+                           '-d', 'test', 'bla', 'voff', 'ga'])
+    output = self.buf.getvalue()
+    self.failUnless(output.count('disconnected'))
+    self.failUnless(output.count("'test'") == 0)  # tool name doesn't occur
+    self.failUnless(output.count('bla'))
+    self.failUnless(output.count('simple-input.xml'))
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Package 'grit.node'
+'''
+
+pass

tools/grit/grit/node/base.py

@@ -0,0 +1,548 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Base types for nodes in a GRIT resource tree.
+'''
+
+import os
+import types
+from xml.sax import saxutils
+
+from grit import exception
+from grit import util
+from grit import clique
+import grit.format.interface
+
+
+class Node(grit.format.interface.ItemFormatter):
+  '''An item in the tree that has children.  Also implements the
+  ItemFormatter interface to allow formatting a node as a GRD document.'''
+
+  # Valid content types that can be returned by _ContentType()
+  _CONTENT_TYPE_NONE = 0   # No CDATA content but may have children
+  _CONTENT_TYPE_CDATA = 1  # Only CDATA, no children.
+  _CONTENT_TYPE_MIXED = 2  # CDATA and children, possibly intermingled
+  
+  def __init__(self):
+    self.children = []        # A list of child elements
+    self.mixed_content = []   # A list of u'' and/or child elements (this
+                              # duplicates 'children' but
+                              # is needed to preserve markup-type content).
+    self.name = u''           # The name of this element
+    self.attrs = {}           # The set of attributes (keys to values)
+    self.parent = None        # Our parent unless we are the root element.
+    self.uberclique = None    # Allows overriding uberclique for parts of tree
+  
+  def __iter__(self):
+    '''An in-order iteration through the tree that this node is the
+    root of.'''
+    return self.inorder()
+  
+  def inorder(self):
+    '''Generator that generates first this node, then the same generator for
+    any child nodes.'''
+    yield self
+    for child in self.children:
+      for iterchild in child.inorder():
+        yield iterchild
+  
+  def GetRoot(self):
+    '''Returns the root Node in the tree this Node belongs to.'''
+    curr = self
+    while curr.parent:
+      curr = curr.parent
+    return curr
+  
+    # TODO(joi) Use this (currently untested) optimization?:
+    #if hasattr(self, '_root'):
+    #  return self._root
+    #curr = self
+    #while curr.parent and not hasattr(curr, '_root'):
+    #  curr = curr.parent
+    #if curr.parent:
+    #  self._root = curr._root
+    #else:
+    #  self._root = curr
+    #return self._root
+    
+  def StartParsing(self, name, parent):
+    '''Called at the start of parsing.
+    
+    Args:
+      name: u'elementname'
+      parent: grit.node.base.Node or subclass or None
+    '''
+    assert isinstance(name, types.StringTypes)
+    assert not parent or isinstance(parent, Node)
+    self.name = name
+    self.parent = parent
+
+  def AddChild(self, child):
+    '''Adds a child to the list of children of this node, if it is a valid
+    child for the node.'''
+    assert isinstance(child, Node)
+    if (not self._IsValidChild(child) or
+        self._ContentType() == self._CONTENT_TYPE_CDATA):
+      if child.parent:
+        explanation = 'child %s of parent %s' % (child.name, child.parent.name)
+      else:
+        explanation = 'node %s with no parent' % child.name
+      raise exception.UnexpectedChild(explanation)
+    self.children.append(child)
+    self.mixed_content.append(child)
+
+  def RemoveChild(self, child_id):
+    '''Removes the first node that has a "name" attribute which
+    matches "child_id" in the list of immediate children of
+    this node.
+
+    Args:
+      child_id: String identifying the child to be removed
+    '''
+    index = 0
+    # Safe not to copy since we only remove the first element found
+    for child in self.children:
+      name_attr = child.attrs['name']
+      if name_attr == child_id:
+        self.children.pop(index)
+        self.mixed_content.pop(index)
+        break
+      index += 1
+      
+  def AppendContent(self, content):
+    '''Appends a chunk of text as content of this node.
+    
+    Args:
+      content: u'hello'
+    
+    Return:
+      None
+    '''
+    assert isinstance(content, types.StringTypes)
+    if self._ContentType() != self._CONTENT_TYPE_NONE:
+      self.mixed_content.append(content)
+    elif content.strip() != '':
+      raise exception.UnexpectedContent()
+    
+  def HandleAttribute(self, attrib, value):
+    '''Informs the node of an attribute that was parsed out of the GRD file
+    for it.
+    
+    Args:
+      attrib: 'name'
+      value: 'fooblat'
+    
+    Return:
+      None
+    '''
+    assert isinstance(attrib, types.StringTypes)
+    assert isinstance(value, types.StringTypes)
+    if self._IsValidAttribute(attrib, value):
+      self.attrs[attrib] = value
+    else:
+      raise exception.UnexpectedAttribute(attrib)
+  
+  def EndParsing(self):
+    '''Called at the end of parsing.'''
+    
+    # TODO(joi) Rewrite this, it's extremely ugly!
+    if len(self.mixed_content):
+      if isinstance(self.mixed_content[0], types.StringTypes):
+        # Remove leading and trailing chunks of pure whitespace.
+        while (len(self.mixed_content) and
+               isinstance(self.mixed_content[0], types.StringTypes) and
+               self.mixed_content[0].strip() == ''):
+          self.mixed_content = self.mixed_content[1:]
+        # Strip leading and trailing whitespace from mixed content chunks
+        # at front and back.
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[0], types.StringTypes)):
+          self.mixed_content[0] = self.mixed_content[0].lstrip()
+        # Remove leading and trailing ''' (used to demarcate whitespace)
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[0], types.StringTypes)):
+          if self.mixed_content[0].startswith("'''"):
+            self.mixed_content[0] = self.mixed_content[0][3:]
+    if len(self.mixed_content):
+      if isinstance(self.mixed_content[-1], types.StringTypes):
+        # Same stuff all over again for the tail end.
+        while (len(self.mixed_content) and
+               isinstance(self.mixed_content[-1], types.StringTypes) and
+               self.mixed_content[-1].strip() == ''):
+          self.mixed_content = self.mixed_content[:-1]
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[-1], types.StringTypes)):
+          self.mixed_content[-1] = self.mixed_content[-1].rstrip()
+        if (len(self.mixed_content) and
+            isinstance(self.mixed_content[-1], types.StringTypes)):
+          if self.mixed_content[-1].endswith("'''"):
+            self.mixed_content[-1] = self.mixed_content[-1][:-3]
+    
+    # Check that all mandatory attributes are there.
+    for node_mandatt in self.MandatoryAttributes():
+      mandatt_list = [] 
+      if node_mandatt.find('|') >= 0:
+        mandatt_list = node_mandatt.split('|')
+      else:
+        mandatt_list.append(node_mandatt)
+      
+      mandatt_option_found = False
+      for mandatt in mandatt_list:
+        assert mandatt not in self.DefaultAttributes().keys()
+        if mandatt in self.attrs:
+          if not mandatt_option_found:
+            mandatt_option_found = True
+          else:
+            raise exception.MutuallyExclusiveMandatoryAttribute(mandatt)
+          
+      if not mandatt_option_found:   
+        raise exception.MissingMandatoryAttribute(mandatt)
+    
+    # Add default attributes if not specified in input file.
+    for defattr in self.DefaultAttributes():
+      if not defattr in self.attrs:
+        self.attrs[defattr] = self.DefaultAttributes()[defattr]
+  
+  def GetCdata(self):
+    '''Returns all CDATA of this element, concatenated into a single
+    string.  Note that this ignores any elements embedded in CDATA.'''
+    return ''.join(filter(lambda c: isinstance(c, types.StringTypes),
+                          self.mixed_content))
+  
+  def __unicode__(self):
+    '''Returns this node and all nodes below it as an XML document in a Unicode
+    string.'''
+    header = u'<?xml version="1.0" encoding="UTF-8"?>\n'
+    return header + self.FormatXml()
+  
+  # Compliance with ItemFormatter interface.
+  def Format(self, item, lang_re = None, begin_item=True):
+    if not begin_item:
+      return ''
+    else:
+      return item.FormatXml()
+    
+  def FormatXml(self, indent = u'', one_line = False):
+    '''Returns this node and all nodes below it as an XML
+    element in a Unicode string.  This differs from __unicode__ in that it does
+    not include the <?xml> stuff at the top of the string.  If one_line is true,
+    children and CDATA are layed out in a way that preserves internal
+    whitespace.
+    '''
+    assert isinstance(indent, types.StringTypes)
+    
+    content_one_line = (one_line or
+                        self._ContentType() == self._CONTENT_TYPE_MIXED)
+    inside_content = self.ContentsAsXml(indent, content_one_line)
+    
+    # Then the attributes for this node.
+    attribs = u' '
+    for (attrib, value) in self.attrs.iteritems():
+      # Only print an attribute if it is other than the default value.
+      if (not self.DefaultAttributes().has_key(attrib) or
+          value != self.DefaultAttributes()[attrib]):
+        attribs += u'%s=%s ' % (attrib, saxutils.quoteattr(value))
+    attribs = attribs.rstrip()  # if no attribs, we end up with '', otherwise
+                                # we end up with a space-prefixed string
+    
+    # Finally build the XML for our node and return it
+    if len(inside_content) > 0:
+      if one_line:
+        return u'<%s%s>%s</%s>' % (self.name, attribs, inside_content, self.name)
+      elif content_one_line:
+        return u'%s<%s%s>\n%s  %s\n%s</%s>' % (
+          indent, self.name, attribs,
+          indent, inside_content,
+          indent, self.name)
+      else:
+        return u'%s<%s%s>\n%s\n%s</%s>' % (
+          indent, self.name, attribs,
+          inside_content,
+          indent, self.name)
+    else:
+      return u'%s<%s%s />' % (indent, self.name, attribs)
+  
+  def ContentsAsXml(self, indent, one_line):
+    '''Returns the contents of this node (CDATA and child elements) in XML
+    format.  If 'one_line' is true, the content will be laid out on one line.'''
+    assert isinstance(indent, types.StringTypes)
+    
+    # Build the contents of the element.
+    inside_parts = []
+    last_item = None
+    for mixed_item in self.mixed_content:
+      if isinstance(mixed_item, Node):
+        inside_parts.append(mixed_item.FormatXml(indent + u'  ', one_line))
+        if not one_line:
+          inside_parts.append(u'\n')
+      else:
+        message = mixed_item
+        # If this is the first item and it starts with whitespace, we add
+        # the ''' delimiter.
+        if not last_item and message.lstrip() != message:
+          message = u"'''" + message
+        inside_parts.append(util.EncodeCdata(message))
+      last_item = mixed_item
+
+    # If there are only child nodes and no cdata, there will be a spurious
+    # trailing \n
+    if len(inside_parts) and inside_parts[-1] == '\n':
+      inside_parts = inside_parts[:-1]
+    
+    # If the last item is a string (not a node) and ends with whitespace,
+    # we need to add the ''' delimiter.
+    if (isinstance(last_item, types.StringTypes) and
+        last_item.rstrip() != last_item):
+      inside_parts[-1] = inside_parts[-1] + u"'''"
+
+    return u''.join(inside_parts)
+  
+  def RunGatherers(self, recursive=0, debug=False):
+    '''Runs all gatherers on this object, which may add to the data stored
+    by the object.  If 'recursive' is true, will call RunGatherers() recursively
+    on all child nodes first.  If 'debug' is True, will print out information
+    as it is running each nodes' gatherers.
+    
+    Gatherers for <translations> child nodes will always be run after all other
+    child nodes have been gathered.
+    '''
+    if recursive:
+      process_last = []
+      for child in self.children:
+        if child.name == 'translations':
+          process_last.append(child)
+        else:
+          child.RunGatherers(recursive=recursive, debug=debug)
+      for child in process_last:
+        child.RunGatherers(recursive=recursive, debug=debug)
+  
+  def ItemFormatter(self, type):
+    '''Returns an instance of the item formatter for this object of the
+    specified type, or None if not supported.
+    
+    Args:
+      type: 'rc-header'
+    
+    Return:
+      (object RcHeaderItemFormatter)
+    '''
+    if type == 'xml':
+      return self
+    else:
+      return None
+  
+  def SatisfiesOutputCondition(self):
+    '''Returns true if this node is either not a child of an <if> element
+    or if it is a child of an <if> element and the conditions for it being
+    output are satisfied.
+    
+    Used to determine whether to return item formatters for formats that
+    obey conditional output of resources (e.g. the RC formatters).
+    '''
+    from grit.node import misc
+    if not self.parent or not isinstance(self.parent, misc.IfNode):
+      return True
+    else:
+      return self.parent.IsConditionSatisfied()
+
+  def _IsValidChild(self, child):
+    '''Returns true if 'child' is a valid child of this node.
+    Overridden by subclasses.'''
+    return False
+
+  def _IsValidAttribute(self, name, value):
+    '''Returns true if 'name' is the name of a valid attribute of this element
+    and 'value' is a valid value for that attribute.  Overriden by
+    subclasses unless they have only mandatory attributes.'''
+    return (name in self.MandatoryAttributes() or
+            name in self.DefaultAttributes())
+  
+  def _ContentType(self):
+    '''Returns the type of content this element can have.  Overridden by
+    subclasses.  The content type can be one of the _CONTENT_TYPE_XXX constants
+    above.'''
+    return self._CONTENT_TYPE_NONE
+
+  def MandatoryAttributes(self):
+    '''Returns a list of attribute names that are mandatory (non-optional)
+    on the current element. One can specify a list of 
+    "mutually exclusive mandatory" attributes by specifying them as one
+    element in the list, separated by a "|" character.
+    '''
+    return []
+
+  def DefaultAttributes(self):
+    '''Returns a dictionary of attribute names that have defaults, mapped to
+    the default value.  Overridden by subclasses.'''
+    return {}
+  
+  def GetCliques(self):
+    '''Returns all MessageClique objects belonging to this node.  Overridden
+    by subclasses.
+    
+    Return:
+      [clique1, clique2] or []
+    '''
+    return []
+
+  def ToRealPath(self, path_from_basedir):
+    '''Returns a real path (which can be absolute or relative to the current
+    working directory), given a path that is relative to the base directory
+    set for the GRIT input file.
+    
+    Args:
+      path_from_basedir: '..'
+    
+    Return:
+      'resource'
+    '''
+    return util.normpath(os.path.join(self.GetRoot().GetBaseDir(),
+                                      path_from_basedir))
+
+  def FilenameToOpen(self):
+    '''Returns a path, either absolute or relative to the current working
+    directory, that points to the file the node refers to.  This is only valid
+    for nodes that have a 'file' or 'path' attribute.  Note that the attribute
+    is a path to the file relative to the 'base-dir' of the .grd file, whereas
+    this function returns a path that can be used to open the file.'''
+    file_attribute = 'file'
+    if not file_attribute in self.attrs:
+      file_attribute = 'path'
+    return self.ToRealPath(self.attrs[file_attribute])
+
+  def UberClique(self):
+    '''Returns the uberclique that should be used for messages originating in
+    a given node.  If the node itself has its uberclique set, that is what we
+    use, otherwise we search upwards until we find one.  If we do not find one
+    even at the root node, we set the root node's uberclique to a new
+    uberclique instance.
+    '''
+    node = self
+    while not node.uberclique and node.parent:
+      node = node.parent
+    if not node.uberclique:
+      node.uberclique = clique.UberClique()
+    return node.uberclique
+  
+  def IsTranslateable(self):
+    '''Returns false if the node has contents that should not be translated,
+    otherwise returns false (even if the node has no contents).
+    '''
+    if not 'translateable' in self.attrs:
+      return True
+    else:
+      return self.attrs['translateable'] == 'true'
+
+  def GetNodeById(self, id):
+    '''Returns the node in the subtree parented by this node that has a 'name'
+    attribute matching 'id'.  Returns None if no such node is found.
+    '''
+    for node in self:
+      if 'name' in node.attrs and node.attrs['name'] == id:
+        return node
+    return None
+
+  def GetTextualIds(self):
+    '''Returns the textual ids of this node, if it has some.
+    Otherwise it just returns None.
+    '''
+    if 'name' in self.attrs:
+      return [self.attrs['name']]
+    return None  
+
+  def EvaluateCondition(self, expr):
+    '''Returns true if and only if the Python expression 'expr' evaluates
+    to true.
+    
+    The expression is given a few local variables:
+      - 'lang' is the language currently being output
+      - 'defs' is a map of C preprocessor-style define names to their values
+      - 'pp_ifdef(define)' which behaves just like the C preprocessors #ifdef,
+        i.e. it is shorthand for "define in defs"
+      - 'pp_if(define)' which behaves just like the C preprocessor's #if, i.e.
+        it is shorthand for "define in defs and defs[define]".
+    '''
+    root = self.GetRoot()
+    lang = ''
+    defs = {}
+    def pp_ifdef(define):
+      return define in defs
+    def pp_if(define):
+      return define in defs and defs[define]
+    if hasattr(root, 'output_language'):
+      lang = root.output_language
+    if hasattr(root, 'defines'):
+      defs = root.defines
+    return eval(expr, {},
+                {'lang' : lang,
+                 'defs' : defs,
+                 'pp_ifdef' : pp_ifdef,
+                 'pp_if' : pp_if})
+  
+  def OnlyTheseTranslations(self, languages):
+    '''Turns off loading of translations for languages not in the provided list.
+    
+    Attrs:
+      languages: ['fr', 'zh_cn']
+    '''
+    for node in self:
+      if (hasattr(node, 'IsTranslation') and
+          node.IsTranslation() and
+          node.GetLang() not in languages):
+        node.DisableLoading()
+  
+  def PseudoIsAllowed(self):
+    '''Returns true if this node is allowed to use pseudo-translations.  This
+    is true by default, unless this node is within a <release> node that has
+    the allow_pseudo attribute set to false.
+    '''
+    p = self.parent
+    while p:
+      if 'allow_pseudo' in p.attrs:
+        return (p.attrs['allow_pseudo'].lower() == 'true')
+      p = p.parent
+    return True
+  
+  def ShouldFallbackToEnglish(self):
+    '''Returns true iff this node should fall back to English when
+    pseudotranslations are disabled and no translation is available for a
+    given message.
+    '''
+    p = self.parent
+    while p:
+      if 'fallback_to_english' in p.attrs:
+        return (p.attrs['fallback_to_english'].lower() == 'true')
+      p = p.parent
+    return False
+
+class ContentNode(Node):
+  '''Convenience baseclass for nodes that can have content.'''
+  def _ContentType(self):
+    return self._CONTENT_TYPE_MIXED

tools/grit/grit/node/base_unittest.py

@@ -0,0 +1,193 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for base.Node functionality (as used in various subclasses)'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import unittest
+
+from grit.node import base
+from grit.node import message
+from grit.node import structure
+from grit.node import variant
+
+def MakePlaceholder(phname='BINGO'):
+  ph = message.PhNode()
+  ph.StartParsing(u'ph', None)
+  ph.HandleAttribute(u'name', phname)
+  ph.AppendContent(u'bongo')
+  ph.EndParsing()
+  return ph
+
+
+class NodeUnittest(unittest.TestCase):
+  def testWhitespaceHandling(self):
+    # We test using the Message node type.
+    node = message.MessageNode()
+    node.StartParsing(u'hello', None)
+    node.HandleAttribute(u'name', u'bla')
+    node.AppendContent(u" '''  two spaces  ")
+    node.EndParsing()
+    self.failUnless(node.GetCdata() == u'  two spaces')
+    
+    node = message.MessageNode()
+    node.StartParsing(u'message', None)
+    node.HandleAttribute(u'name', u'bla')
+    node.AppendContent(u"  two spaces  '''  ")
+    node.EndParsing()
+    self.failUnless(node.GetCdata() == u'two spaces  ')
+
+  def testWhitespaceHandlingWithChildren(self):
+    # We test using the Message node type.
+    node = message.MessageNode()
+    node.StartParsing(u'message', None)
+    node.HandleAttribute(u'name', u'bla')
+    node.AppendContent(u" '''  two spaces  ")
+    node.AddChild(MakePlaceholder())
+    node.AppendContent(u' space before and after ')
+    node.AddChild(MakePlaceholder('BONGO'))
+    node.AppendContent(u" space before two after  '''")
+    node.EndParsing()
+    self.failUnless(node.mixed_content[0] == u'  two spaces  ')
+    self.failUnless(node.mixed_content[2] == u' space before and after ')
+    self.failUnless(node.mixed_content[-1] == u' space before two after  ')
+
+  def testXmlFormatMixedContent(self):
+    # Again test using the Message node type, because it is the only mixed
+    # content node.
+    node = message.MessageNode()
+    node.StartParsing(u'message', None)
+    node.HandleAttribute(u'name', u'name')
+    node.AppendContent(u'Hello <young> ')
+    
+    ph = message.PhNode()
+    ph.StartParsing(u'ph', None)
+    ph.HandleAttribute(u'name', u'USERNAME')
+    ph.AppendContent(u'$1')
+    ex = message.ExNode()
+    ex.StartParsing(u'ex', None)
+    ex.AppendContent(u'Joi')
+    ex.EndParsing()
+    ph.AddChild(ex)
+    ph.EndParsing()
+    
+    node.AddChild(ph)
+    node.EndParsing()
+    
+    non_indented_xml = node.Format(node)
+    self.failUnless(non_indented_xml == u'<message name="name">\n  Hello '
+                    u'&lt;young&gt; <ph name="USERNAME">$1<ex>Joi</ex></ph>'
+                    u'\n</message>')
+    
+    indented_xml = node.FormatXml(u'  ')
+    self.failUnless(indented_xml == u'  <message name="name">\n    Hello '
+                    u'&lt;young&gt; <ph name="USERNAME">$1<ex>Joi</ex></ph>'
+                    u'\n  </message>')
+
+  def testXmlFormatMixedContentWithLeadingWhitespace(self):
+    # Again test using the Message node type, because it is the only mixed
+    # content node.
+    node = message.MessageNode()
+    node.StartParsing(u'message', None)
+    node.HandleAttribute(u'name', u'name')
+    node.AppendContent(u"'''   Hello <young> ")
+    
+    ph = message.PhNode()
+    ph.StartParsing(u'ph', None)
+    ph.HandleAttribute(u'name', u'USERNAME')
+    ph.AppendContent(u'$1')
+    ex = message.ExNode()
+    ex.StartParsing(u'ex', None)
+    ex.AppendContent(u'Joi')
+    ex.EndParsing()
+    ph.AddChild(ex)
+    ph.EndParsing()
+    
+    node.AddChild(ph)
+    node.AppendContent(u" yessiree '''")
+    node.EndParsing()
+    
+    non_indented_xml = node.Format(node)
+    self.failUnless(non_indented_xml ==
+                    u"<message name=\"name\">\n  '''   Hello"
+                    u' &lt;young&gt; <ph name="USERNAME">$1<ex>Joi</ex></ph>'
+                    u" yessiree '''\n</message>")
+    
+    indented_xml = node.FormatXml(u'  ')
+    self.failUnless(indented_xml ==
+                    u"  <message name=\"name\">\n    '''   Hello"
+                    u' &lt;young&gt; <ph name="USERNAME">$1<ex>Joi</ex></ph>'
+                    u" yessiree '''\n  </message>")
+    
+    self.failUnless(node.GetNodeById('name'))
+  
+  def testXmlFormatContentWithEntities(self):
+    '''Tests a bug where &nbsp; would not be escaped correctly.'''
+    from grit import tclib
+    msg_node = message.MessageNode.Construct(None, tclib.Message(
+      text = 'BEGIN_BOLDHelloWHITESPACEthere!END_BOLD Bingo!',
+      placeholders = [
+        tclib.Placeholder('BEGIN_BOLD', '<b>', 'bla'),
+        tclib.Placeholder('WHITESPACE', '&nbsp;', 'bla'),
+        tclib.Placeholder('END_BOLD', '</b>', 'bla')]),
+                                             'BINGOBONGO')
+    xml = msg_node.FormatXml()
+    self.failUnless(xml.find('&nbsp;') == -1, 'should have no entities')
+    
+  def testIter(self):
+    # First build a little tree of message and ph nodes.
+    node = message.MessageNode()
+    node.StartParsing(u'message', None)
+    node.HandleAttribute(u'name', u'bla')
+    node.AppendContent(u" '''  two spaces  ")
+    node.AppendContent(u' space before and after ')
+    ph = message.PhNode()
+    ph.StartParsing(u'ph', None)
+    ph.AddChild(message.ExNode())
+    ph.HandleAttribute(u'name', u'BINGO')
+    ph.AppendContent(u'bongo')
+    node.AddChild(ph)
+    node.AddChild(message.PhNode())
+    node.AppendContent(u" space before two after  '''")
+    
+    order = [message.MessageNode, message.PhNode, message.ExNode, message.PhNode]
+    for n in node:
+      self.failUnless(type(n) == order[0])
+      order = order[1:]
+    self.failUnless(len(order) == 0)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/custom/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/python2.4
+# Copyright 2004 Google Inc.
+# All Rights Reserved.
+# Author: Joi Sigurdsson <joi@google.com>
+
+'''Package 'grit.node.custom'
+'''
+
+pass

tools/grit/grit/node/custom/filename.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''A CustomType for filenames.'''
+
+import re
+
+from grit import clique
+
+
+class WindowsFilename(clique.CustomType):
+  '''Validates that messages can be used as Windows filenames, and strips
+  illegal characters out of translations.
+  '''
+  
+  BANNED = re.compile('\+|:|\/|\\\\|\*|\?|\"|\<|\>|\|')
+  
+  def Validate(self, message):
+    return not self.BANNED.search(message.GetPresentableContent())
+  
+  def ValidateAndModify(self, lang, translation):
+    is_ok = self.Validate(translation)
+    self.ModifyEachTextPart(lang, translation)
+    return is_ok
+  
+  def ModifyTextPart(self, lang, text):
+    return self.BANNED.sub(' ', text)

tools/grit/grit/node/custom/filename_unittest.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.node.custom.filename'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
+
+import unittest
+from grit.node.custom import filename
+from grit import clique
+from grit import tclib
+
+
+class WindowsFilenameUnittest(unittest.TestCase):
+  
+  def testValidate(self):
+    factory = clique.UberClique()
+    msg = tclib.Message(text='Bingo bongo')
+    c = factory.MakeClique(msg)
+    c.SetCustomType(filename.WindowsFilename())
+    translation = tclib.Translation(id=msg.GetId(), text='Bilingo bolongo:')
+    c.AddTranslation(translation, 'fr')
+    self.failUnless(c.MessageForLanguage('fr').GetRealContent() == 'Bilingo bolongo ')
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/empty.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Container nodes that don't have any logic.
+'''
+
+
+from grit.node import base
+from grit.node import include
+from grit.node import structure
+from grit.node import message
+from grit.node import io
+from grit.node import misc
+
+
+class GroupingNode(base.Node):
+  '''Base class for all the grouping elements (<structures>, <includes>,
+  <messages> and <identifiers>).'''
+  def DefaultAttributes(self):
+    return {
+      'first_id' : '',
+      'comment' : '',
+      'fallback_to_english' : 'false',
+    }
+
+
+class IncludesNode(GroupingNode):
+  '''The <includes> element.'''
+  def _IsValidChild(self, child):
+    return isinstance(child, (include.IncludeNode, misc.IfNode))
+
+
+class MessagesNode(GroupingNode):
+  '''The <messages> element.'''
+  def _IsValidChild(self, child):
+    return isinstance(child, (message.MessageNode, misc.IfNode))
+  
+  def ItemFormatter(self, t):
+    '''Return the stringtable itemformatter if an RC is being formatted.'''
+    if t in ['rc_all', 'rc_translateable', 'rc_nontranslateable']:
+      from grit.format import rc  # avoid circular dep by importing here
+      return rc.StringTable()
+
+
+class StructuresNode(GroupingNode):
+  '''The <structures> element.'''
+  def _IsValidChild(self, child):
+    return isinstance(child, (structure.StructureNode, misc.IfNode))
+
+
+class TranslationsNode(base.Node):
+  '''The <translations> element.'''
+  def _IsValidChild(self, child):
+    return isinstance(child, io.FileNode)
+
+
+class OutputsNode(base.Node):
+  '''The <outputs> element.'''
+  def _IsValidChild(self, child):
+    return isinstance(child, io.OutputNode)
+
+
+class IdentifiersNode(GroupingNode):
+  '''The <identifiers> element.'''
+  def _IsValidChild(self, child):
+    from grit.node import misc
+    return isinstance(child, misc.IdentifierNode)

tools/grit/grit/node/include.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Handling of the <include> element.
+'''
+
+
+import grit.format.rc_header
+import grit.format.rc
+
+from grit.node import base
+from grit import util
+
+class IncludeNode(base.Node):
+  '''An <include> element.'''
+  
+  def _IsValidChild(self, child):
+    return False
+
+  def MandatoryAttributes(self):
+    return ['name', 'type', 'file']
+
+  def DefaultAttributes(self):
+    return {'translateable' : 'true', 
+      'generateid': 'true', 
+      'filenameonly': 'false',
+      'relativepath': 'false',
+      }
+
+  def ItemFormatter(self, t):
+    if t == 'rc_header':
+      return grit.format.rc_header.Item()
+    elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and
+          self.SatisfiesOutputCondition()):
+      return grit.format.rc.RcInclude(self.attrs['type'].upper(), 
+        self.attrs['filenameonly'] == 'true',
+        self.attrs['relativepath'] == 'true')
+    else:
+      return super(type(self), self).ItemFormatter(t)
+  
+  def FileForLanguage(self, lang, output_dir):
+    '''Returns the file for the specified language.  This allows us to return
+    different files for different language variants of the include file.
+    '''
+    return self.FilenameToOpen()
+
+  # static method
+  def Construct(parent, name, type, file, translateable=True, 
+      filenameonly=False, relativepath=False):
+    '''Creates a new node which is a child of 'parent', with attributes set
+    by parameters of the same name.
+    '''
+    # Convert types to appropriate strings
+    translateable = util.BoolToString(translateable)
+    filenameonly = util.BoolToString(filenameonly)
+    relativepath = util.BoolToString(relativepath)
+    
+    node = IncludeNode()
+    node.StartParsing('include', parent)
+    node.HandleAttribute('name', name)
+    node.HandleAttribute('type', type)
+    node.HandleAttribute('file', file)
+    node.HandleAttribute('translateable', translateable)
+    node.HandleAttribute('filenameonly', filenameonly)
+    node.HandleAttribute('relativepath', relativepath)
+    node.EndParsing()
+    return node
+  Construct = staticmethod(Construct)

tools/grit/grit/node/io.py

@@ -0,0 +1,130 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The <output> and <file> elements.
+'''
+
+import os
+import re
+import grit.format.rc_header
+
+from grit.node import base
+from grit import exception
+from grit import util
+from grit import xtb_reader
+
+
+class FileNode(base.Node):
+  '''A <file> element.'''
+  
+  def __init__(self):
+    super(type(self), self).__init__()
+    self.re = None
+    self.should_load_ = True
+  
+  def IsTranslation(self):
+    return True
+  
+  def GetLang(self):
+    return self.attrs['lang']
+  
+  def DisableLoading(self):
+    self.should_load_ = False
+  
+  def MandatoryAttributes(self):
+    return ['path', 'lang']
+  
+  def RunGatherers(self, recursive=False, debug=False):
+    if not self.should_load_:
+      return
+    
+    xtb_file = file(self.GetFilePath())
+    try:
+      lang = xtb_reader.Parse(xtb_file,
+                              self.UberClique().GenerateXtbParserCallback(
+                                self.attrs['lang'], debug=debug))
+    except:
+      print "Exception during parsing of %s" % self.GetFilePath()
+      raise
+    assert (lang == self.attrs['lang'], 'The XTB file you '
+            'reference must contain messages in the language specified\n'
+            'by the \'lang\' attribute.')
+  
+  def GetFilePath(self):
+    return self.ToRealPath(os.path.expandvars(self.attrs['path']))
+
+
+class OutputNode(base.Node):
+  '''An <output> element.'''
+  
+  def MandatoryAttributes(self):
+    return ['filename', 'type']
+  
+  def DefaultAttributes(self):
+    return { 'lang' : '', # empty lang indicates all languages
+             'language_section' : 'neutral' # defines a language neutral section
+             } 
+  
+  def GetType(self):
+    return self.attrs['type']
+
+  def GetLanguage(self):
+    '''Returns the language ID, default 'en'.'''
+    return self.attrs['lang']
+
+  def GetFilename(self):
+    return self.attrs['filename']
+
+  def GetOutputFilename(self):
+    if hasattr(self, 'output_filename'):
+      return self.output_filename
+    else:
+      return self.attrs['filename']
+
+  def _IsValidChild(self, child):
+    return isinstance(child, EmitNode)
+
+class EmitNode(base.ContentNode):
+  ''' An <emit> element.'''
+
+  def DefaultAttributes(self):
+    return { 'emit_type' : 'prepend'}
+
+  def GetEmitType(self):
+    '''Returns the emit_type for this node. Default is 'append'.'''
+    return self.attrs['emit_type']
+
+  def ItemFormatter(self, t):
+    if t == 'rc_header':
+      return grit.format.rc_header.EmitAppender() 
+    else:
+      return super(type(self), self).ItemFormatter(t) 
+  
+

tools/grit/grit/node/io_unittest.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for io.FileNode'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import os
+import StringIO
+import unittest
+
+from grit.node import misc
+from grit.node import io
+from grit.node import empty
+from grit import grd_reader
+from grit import util
+
+
+class FileNodeUnittest(unittest.TestCase):
+  def testGetPath(self):
+    root = misc.GritNode()
+    root.StartParsing(u'grit', None)
+    root.HandleAttribute(u'latest_public_release', u'0')
+    root.HandleAttribute(u'current_release', u'1')
+    root.HandleAttribute(u'base_dir', ur'..\resource')
+    translations = empty.TranslationsNode()
+    translations.StartParsing(u'translations', root)
+    root.AddChild(translations)
+    file_node = io.FileNode()
+    file_node.StartParsing(u'file', translations)
+    file_node.HandleAttribute(u'path', ur'flugel\kugel.pdf')
+    translations.AddChild(file_node)
+    root.EndParsing()
+    
+    self.failUnless(file_node.GetFilePath() ==
+                    util.normpath(
+                      os.path.join(ur'../resource', ur'flugel/kugel.pdf')))
+  
+  def testLoadTranslations(self):
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <translations>
+          <file path="fr.xtb" lang="fr" />
+        </translations>
+        <release seq="3">
+          <messages>
+            <message name="ID_HELLO">Hello!</message>
+            <message name="ID_HELLO_USER">Hello <ph name="USERNAME">%s<ex>Joi</ex></ph></message>
+          </messages>
+        </release>
+      </grit>'''), util.PathFromRoot('grit/test/data'))
+    grd.RunGatherers(recursive=True)
+    self.failUnless(True)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/mapping.py

@@ -0,0 +1,82 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Maps each node type to an implementation class.
+When adding a new node type, you add to this mapping.
+'''
+
+
+from grit import exception
+
+from grit.node import empty
+from grit.node import message
+from grit.node import misc
+from grit.node import variant
+from grit.node import structure
+from grit.node import include
+from grit.node import io
+
+
+_ELEMENT_TO_CLASS = {
+  'includes'      : empty.IncludesNode,
+  'messages'      : empty.MessagesNode,
+  'structures'    : empty.StructuresNode,
+  'translations'  : empty.TranslationsNode,
+  'outputs'       : empty.OutputsNode,
+  'message'       : message.MessageNode,
+  'ph'            : message.PhNode,
+  'ex'            : message.ExNode,
+  'grit'          : misc.GritNode,
+  'include'       : include.IncludeNode,
+  'structure'     : structure.StructureNode,
+  'skeleton'      : variant.SkeletonNode,
+  'release'       : misc.ReleaseNode,
+  'file'          : io.FileNode,
+  'output'        : io.OutputNode,
+  'emit'          : io.EmitNode,
+  'identifiers'   : empty.IdentifiersNode,
+  'identifier'    : misc.IdentifierNode,
+  'if'            : misc.IfNode,
+}
+
+
+def ElementToClass(name, typeattr):
+  '''Maps an element to a class that handles the element.
+  
+  Args:
+    name: 'element' (the name of the element)
+    typeattr: 'type' (the value of the type attribute, if present, else None)
+  
+  Return:
+    type
+  '''
+  if not _ELEMENT_TO_CLASS.has_key(name):
+    raise exception.UnknownElement()
+  return _ELEMENT_TO_CLASS[name]

tools/grit/grit/node/message.py

@@ -0,0 +1,271 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Handling of the <message> element.
+'''
+
+import re
+import types
+
+from grit.node import base
+
+import grit.format.rc_header
+import grit.format.rc
+
+from grit import clique
+from grit import exception
+from grit import tclib
+from grit import util
+
+
+# Finds whitespace at the start and end of a string which can be multiline.
+_WHITESPACE = re.compile('(?P<start>\s*)(?P<body>.+?)(?P<end>\s*)\Z',
+                         re.DOTALL | re.MULTILINE)
+
+
+class MessageNode(base.ContentNode):
+  '''A <message> element.'''
+  
+  # For splitting a list of things that can be separated by commas or
+  # whitespace
+  _SPLIT_RE = re.compile('\s*,\s*|\s+')
+  
+  def __init__(self):
+    super(type(self), self).__init__()
+    # Valid after EndParsing, this is the MessageClique that contains the
+    # source message and any translations of it that have been loaded.
+    self.clique = None
+    
+    # We don't send leading and trailing whitespace into the translation
+    # console, but rather tack it onto the source message and any
+    # translations when formatting them into RC files or what have you.
+    self.ws_at_start = ''  # Any whitespace characters at the start of the text
+    self.ws_at_end = ''  # --"-- at the end of the text
+    
+    # A list of "shortcut groups" this message is in.  We check to make sure
+    # that shortcut keys (e.g. &J) within each shortcut group are unique.
+    self.shortcut_groups_ = []
+
+  def _IsValidChild(self, child):
+    return isinstance(child, (PhNode))
+
+  def _IsValidAttribute(self, name, value):
+    if name not in ['name', 'offset', 'translateable', 'desc', 'meaning',
+                    'internal_comment', 'shortcut_groups', 'custom_type',
+                    'validation_expr']:
+      return False
+    if name == 'translateable' and value not in ['true', 'false']:
+      return False
+    return True
+  
+  def MandatoryAttributes(self):
+    return ['name|offset']
+  
+  def DefaultAttributes(self):
+    return {
+      'translateable' : 'true',
+      'desc' : '',
+      'meaning' : '',
+      'internal_comment' : '',
+      'shortcut_groups' : '',
+      'custom_type' : '',
+      'validation_expr' : '',
+    }
+
+  def GetTextualIds(self):
+    '''
+    Returns the concatenation of the parent's node first_id and
+    this node's offset if it has one, otherwise just call the
+    superclass' implementation
+    '''
+    if 'offset' in self.attrs:
+      # we search for the first grouping node in the parents' list
+      # to take care of the case where the first parent is an <if> node
+      grouping_parent = self.parent
+      import grit.node.empty
+      while grouping_parent and not isinstance(grouping_parent,
+                                               grit.node.empty.GroupingNode):
+        grouping_parent = grouping_parent.parent
+      
+      assert 'first_id' in grouping_parent.attrs
+      return [grouping_parent.attrs['first_id'] + '_' + self.attrs['offset']]
+    else:
+      return super(type(self), self).GetTextualIds()
+  
+  def IsTranslateable(self):
+    return self.attrs['translateable'] == 'true'
+
+  def ItemFormatter(self, t):
+    if t == 'rc_header':
+      return grit.format.rc_header.Item()
+    elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and
+          self.SatisfiesOutputCondition()):
+      return grit.format.rc.Message()
+    else:
+      return super(type(self), self).ItemFormatter(t)
+
+  def EndParsing(self):
+    super(type(self), self).EndParsing()
+    
+    # Make the text (including placeholder references) and list of placeholders,
+    # then strip and store leading and trailing whitespace and create the
+    # tclib.Message() and a clique to contain it.
+    
+    text = ''
+    placeholders = []
+    for item in self.mixed_content:
+      if isinstance(item, types.StringTypes):
+        text += item
+      else:
+        presentation = item.attrs['name'].upper()
+        text += presentation
+        ex = ' '
+        if len(item.children):
+          ex = item.children[0].GetCdata()
+        original = item.GetCdata()
+        placeholders.append(tclib.Placeholder(presentation, original, ex))
+    
+    m = _WHITESPACE.match(text)
+    if m:
+      self.ws_at_start = m.group('start')
+      self.ws_at_end = m.group('end')
+      text = m.group('body')
+    
+    self.shortcut_groups_ = self._SPLIT_RE.split(self.attrs['shortcut_groups'])
+    self.shortcut_groups_ = [i for i in self.shortcut_groups_ if i != '']
+    
+    description_or_id = self.attrs['desc']
+    if description_or_id == '' and 'name' in self.attrs:
+      description_or_id = 'ID: %s' % self.attrs['name']
+    
+    message = tclib.Message(text=text, placeholders=placeholders,
+                            description=description_or_id,
+                            meaning=self.attrs['meaning'])
+    self.clique = self.UberClique().MakeClique(message, self.IsTranslateable())
+    for group in self.shortcut_groups_:
+      self.clique.AddToShortcutGroup(group)
+    if self.attrs['custom_type'] != '':
+      self.clique.SetCustomType(util.NewClassInstance(self.attrs['custom_type'],
+                                                      clique.CustomType))
+    elif self.attrs['validation_expr'] != '':
+      self.clique.SetCustomType(
+        clique.OneOffCustomType(self.attrs['validation_expr']))
+      
+  def GetCliques(self):
+    if self.clique:
+      return [self.clique]
+    else:
+      return []
+  
+  def Translate(self, lang):
+    '''Returns a translated version of this message.
+    '''
+    assert self.clique
+    return self.clique.MessageForLanguage(lang,
+                                          self.PseudoIsAllowed(),
+                                          self.ShouldFallbackToEnglish()
+                                          ).GetRealContent()
+  
+  def NameOrOffset(self):
+    if 'name' in self.attrs:
+      return self.attrs['name']
+    else:
+      return self.attrs['offset']
+
+  # static method
+  def Construct(parent, message, name, desc='', meaning='', translateable=True):
+    '''Constructs a new message node that is a child of 'parent', with the
+    name, desc, meaning and translateable attributes set using the same-named
+    parameters and the text of the message and any placeholders taken from
+    'message', which must be a tclib.Message() object.'''
+    # Convert type to appropriate string
+    if translateable:
+      translateable = 'true'
+    else:
+      translateable = 'false'
+    
+    node = MessageNode()
+    node.StartParsing('message', parent)
+    node.HandleAttribute('name', name)
+    node.HandleAttribute('desc', desc)
+    node.HandleAttribute('meaning', meaning)
+    node.HandleAttribute('translateable', translateable)
+    
+    items = message.GetContent()
+    for ix in range(len(items)):
+      if isinstance(items[ix], types.StringTypes):
+        text = items[ix]
+        
+        # Ensure whitespace at front and back of message is correctly handled.
+        if ix == 0:
+          text = "'''" + text
+        if ix == len(items) - 1:
+          text = text + "'''"
+        
+        node.AppendContent(text)
+      else:
+        phnode = PhNode()
+        phnode.StartParsing('ph', node)
+        phnode.HandleAttribute('name', items[ix].GetPresentation())
+        phnode.AppendContent(items[ix].GetOriginal())
+        
+        if len(items[ix].GetExample()) and items[ix].GetExample() != ' ':
+          exnode = ExNode()
+          exnode.StartParsing('ex', phnode)
+          exnode.AppendContent(items[ix].GetExample())
+          exnode.EndParsing()
+          phnode.AddChild(exnode)
+        
+        phnode.EndParsing()
+        node.AddChild(phnode)
+    
+    node.EndParsing()
+    return node
+  Construct = staticmethod(Construct)
+
+class PhNode(base.ContentNode):
+  '''A <ph> element.'''
+  
+  def _IsValidChild(self, child):
+    return isinstance(child, ExNode)
+
+  def MandatoryAttributes(self):
+    return ['name']
+
+  def EndParsing(self):
+    super(type(self), self).EndParsing()
+    # We only allow a single example for each placeholder
+    if len(self.children) > 1:
+      raise exception.TooManyExamples()
+
+
+class ExNode(base.ContentNode):
+  '''An <ex> element.'''
+  pass

tools/grit/grit/node/message_unittest.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.node.message'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import unittest
+import StringIO
+
+from grit.node import message
+from grit import grd_reader
+from grit import tclib
+
+class MessageUnittest(unittest.TestCase):
+  def testMessage(self):
+    buf = StringIO.StringIO('''<message name="IDS_GREETING"
+      desc="Printed to greet the currently logged in user">
+      Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you doing today?
+      </message>''')
+    res = grd_reader.Parse(buf, flexible_root = True)
+    cliques = res.GetCliques()
+    content = cliques[0].GetMessage().GetPresentableContent()
+    self.failUnless(content == 'Hello USERNAME, how are you doing today?')
+  
+  def testMessageWithWhitespace(self):
+    buf = StringIO.StringIO('<message name="IDS_BLA" desc="">'
+                            '\'\'\'  Hello there <ph name="USERNAME">%s</ph>   \'\'\''
+                            '</message>')
+    res = grd_reader.Parse(buf, flexible_root = True)
+    content = res.GetCliques()[0].GetMessage().GetPresentableContent()
+    self.failUnless(content == 'Hello there USERNAME')
+    self.failUnless(res.ws_at_start == '  ')
+    self.failUnless(res.ws_at_end == '   ')
+
+  def testConstruct(self):
+    msg = tclib.Message(text="   Hello USERNAME, how are you?   BINGO\t\t",
+                        placeholders=[tclib.Placeholder('USERNAME', '%s', 'Joi'),
+                                      tclib.Placeholder('BINGO', '%d', '11')])
+    msg_node = message.MessageNode.Construct(None, msg, 'BINGOBONGO')
+    self.failUnless(msg_node.children[0].name == 'ph')
+    self.failUnless(msg_node.children[0].children[0].name == 'ex')
+    self.failUnless(msg_node.children[0].children[0].GetCdata() == 'Joi')
+    self.failUnless(msg_node.children[1].children[0].GetCdata() == '11')
+    self.failUnless(msg_node.ws_at_start == '   ')
+    self.failUnless(msg_node.ws_at_end == '\t\t')
+  
+  def testUnicodeConstruct(self):
+    text = u'Howdie \u00fe'
+    msg = tclib.Message(text=text)
+    msg_node = message.MessageNode.Construct(None, msg, 'BINGOBONGO')
+    msg_from_node = msg_node.GetCdata()
+    self.failUnless(msg_from_node == text)
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/misc.py

@@ -0,0 +1,284 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Miscellaneous node types.
+'''
+
+import os.path
+
+from grit.node import base
+from grit.node import message
+
+from grit import exception
+from grit import constants
+from grit import util
+
+import grit.format.rc_header
+
+
+
+class IfNode(base.Node):
+  '''A node for conditional inclusion of resources.
+  '''
+  
+  def _IsValidChild(self, child):
+    from grit.node import empty
+    assert self.parent, '<if> node should never be root.'
+    if isinstance(self.parent, empty.IncludesNode):
+      from grit.node import include
+      return isinstance(child, include.IncludeNode)
+    elif isinstance(self.parent, empty.MessagesNode):
+      from grit.node import message
+      return isinstance(child, message.MessageNode)
+    elif isinstance(self.parent, empty.StructuresNode):
+      from grit.node import structure
+      return isinstance(child, structure.StructureNode)
+    else:
+      return False
+  
+  def MandatoryAttributes(self):
+    return ['expr']
+  
+  def IsConditionSatisfied(self):
+    '''Returns true if and only if the Python expression stored in attribute
+    'expr' evaluates to true.
+    '''
+    return self.EvaluateCondition(self.attrs['expr'])
+
+
+class ReleaseNode(base.Node):
+  '''The <release> element.'''
+
+  def _IsValidChild(self, child):
+    from grit.node import empty
+    return isinstance(child, (empty.IncludesNode, empty.MessagesNode,
+                              empty.StructuresNode, empty.IdentifiersNode))
+  
+  def _IsValidAttribute(self, name, value):
+    return (
+      (name == 'seq' and int(value) <= self.GetRoot().GetCurrentRelease()) or
+      name == 'allow_pseudo'
+    )
+
+  def MandatoryAttributes(self):
+    return ['seq']
+  
+  def DefaultAttributes(self):
+    return { 'allow_pseudo' : 'true' }
+  
+  def GetReleaseNumber():
+    '''Returns the sequence number of this release.'''
+    return self.attribs['seq']
+
+
+class GritNode(base.Node):
+  '''The <grit> root element.'''
+  
+  def __init__(self):
+    base.Node.__init__(self)
+    self.output_language = ''
+    self.defines = {}
+  
+  def _IsValidChild(self, child):
+    from grit.node import empty
+    return isinstance(child, (ReleaseNode, empty.TranslationsNode,
+                              empty.OutputsNode))
+  
+  def _IsValidAttribute(self, name, value):
+    if name not in ['base_dir', 'source_lang_id',
+                    'latest_public_release', 'current_release',
+                    'enc_check', 'tc_project']:
+      return False
+    if name in ['latest_public_release', 'current_release'] and value.strip(
+      '0123456789') != '':
+      return False
+    return True
+  
+  def MandatoryAttributes(self):
+    return ['latest_public_release', 'current_release']
+  
+  def DefaultAttributes(self):
+    return {
+      'base_dir' : '.',
+      'source_lang_id' : 'en',
+      'enc_check' : constants.ENCODING_CHECK,
+      'tc_project' : 'NEED_TO_SET_tc_project_ATTRIBUTE',
+    }
+
+  def EndParsing(self):
+    base.Node.EndParsing(self)
+    if (int(self.attrs['latest_public_release'])
+        > int(self.attrs['current_release'])):
+      raise exception.Parsing('latest_public_release cannot have a greater '
+                              'value than current_release')
+    
+    self.ValidateUniqueIds()
+    
+    # Add the encoding check if it's not present (should ensure that it's always
+    # present in all .grd files generated by GRIT). If it's present, assert if
+    # it's not correct.
+    if 'enc_check' not in self.attrs or self.attrs['enc_check'] == '':
+      self.attrs['enc_check'] = constants.ENCODING_CHECK
+    else:
+      assert self.attrs['enc_check'] == constants.ENCODING_CHECK, (
+        'Are you sure your .grd file is in the correct encoding (UTF-8)?')
+  
+  def ValidateUniqueIds(self):
+    '''Validate that 'name' attribute is unique in all nodes in this tree
+    except for nodes that are children of <if> nodes.
+    '''
+    unique_names = {}
+    duplicate_names = []
+    for node in self:
+      if isinstance(node, message.PhNode):
+        continue  # PhNode objects have a 'name' attribute which is not an ID
+      
+      node_ids = node.GetTextualIds()
+      if node_ids:
+        for node_id in node_ids:
+          if util.SYSTEM_IDENTIFIERS.match(node_id):
+            continue  # predefined IDs are sometimes used more than once
+        
+          # Don't complain about duplicate IDs if they occur in a node that is
+          # inside an <if> node.
+          if (node_id in unique_names and node_id not in duplicate_names and
+              (not node.parent or not isinstance(node.parent, IfNode))):
+            duplicate_names.append(node_id)
+          unique_names[node_id] = 1
+  
+    if len(duplicate_names):
+      raise exception.DuplicateKey(', '.join(duplicate_names))
+
+  
+  def GetCurrentRelease(self):
+    '''Returns the current release number.'''
+    return int(self.attrs['current_release'])
+  
+  def GetLatestPublicRelease(self):
+    '''Returns the latest public release number.'''
+    return int(self.attrs['latest_public_release'])
+  
+  def GetSourceLanguage(self):
+    '''Returns the language code of the source language.'''
+    return self.attrs['source_lang_id']
+  
+  def GetTcProject(self):
+    '''Returns the name of this project in the TranslationConsole, or
+    'NEED_TO_SET_tc_project_ATTRIBUTE' if it is not defined.'''
+    return self.attrs['tc_project']
+  
+  def SetOwnDir(self, dir):
+    '''Informs the 'grit' element of the directory the file it is in resides.
+    This allows it to calculate relative paths from the input file, which is
+    what we desire (rather than from the current path).
+    
+    Args:
+      dir: r'c:\bla'
+    
+    Return:
+      None
+    '''
+    assert dir
+    self.base_dir = os.path.normpath(os.path.join(dir, self.attrs['base_dir']))
+  
+  def GetBaseDir(self):
+    '''Returns the base directory, relative to the working directory.  To get
+    the base directory as set in the .grd file, use GetOriginalBaseDir()
+    '''
+    if hasattr(self, 'base_dir'):
+      return self.base_dir
+    else:
+      return self.GetOriginalBaseDir()
+  
+  def GetOriginalBaseDir(self):
+    '''Returns the base directory, as set in the .grd file.
+    '''
+    return self.attrs['base_dir']
+    
+  def GetOutputFiles(self):
+    '''Returns the list of <file> nodes that are children of this node's
+    <outputs> child.'''
+    for child in self.children:
+      if child.name == 'outputs':
+        return child.children
+    raise exception.MissingElement()
+
+  def ItemFormatter(self, t):
+    if t == 'rc_header':
+      from grit.format import rc_header  # import here to avoid circular dep
+      return rc_header.TopLevel()
+    elif t in ['rc_all', 'rc_translateable', 'rc_nontranslateable']:
+      from grit.format import rc  # avoid circular dep
+      return rc.TopLevel()
+    else:
+      return super(type(self), self).ItemFormatter(t)
+
+  def SetOutputContext(self, output_language, defines):
+    self.output_language = output_language
+    self.defines = defines
+
+
+class IdentifierNode(base.Node):
+  '''A node for specifying identifiers that should appear in the resource
+  header file, and be unique amongst all other resource identifiers, but don't
+  have any other attributes or reference any resources.
+  '''
+  
+  def MandatoryAttributes(self):
+    return ['name']
+  
+  def DefaultAttributes(self):
+    return { 'comment' : '', 'id' : '' }
+
+  def ItemFormatter(self, t):
+    if t == 'rc_header':
+      return grit.format.rc_header.Item()
+
+  def GetId(self):
+    '''Returns the id of this identifier if it has one, None otherwise
+    '''
+    if 'id' in self.attrs:
+      return self.attrs['id']
+    return None  
+      
+  # static method
+  def Construct(parent, name, id, comment):
+    '''Creates a new node which is a child of 'parent', with attributes set
+    by parameters of the same name.
+    '''
+    node = IdentifierNode()
+    node.StartParsing('identifier', parent)
+    node.HandleAttribute('name', name)
+    node.HandleAttribute('id', id)
+    node.HandleAttribute('comment', comment)
+    node.EndParsing()
+    return node
+  Construct = staticmethod(Construct)
+

tools/grit/grit/node/misc_unittest.py

@@ -0,0 +1,162 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for misc.GritNode'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import unittest
+import StringIO
+
+from grit import grd_reader
+import grit.exception
+from grit import util
+from grit.node import misc
+
+
+class GritNodeUnittest(unittest.TestCase):
+  def testUniqueNameAttribute(self):
+    try:
+      restree = grd_reader.Parse(
+        util.PathFromRoot('grit/test/data/duplicate-name-input.xml'))
+      self.fail('Expected parsing exception because of duplicate names.')
+    except grit.exception.Parsing:
+      pass  # Expected case
+
+
+class IfNodeUnittest(unittest.TestCase):
+  def testIffyness(self):
+    grd = grd_reader.Parse(StringIO.StringIO('''
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <release seq="3">
+          <messages>
+            <if expr="'bingo' in defs">
+              <message name="IDS_BINGO">
+                Bingo!
+              </message>
+            </if>
+            <if expr="'hello' in defs">
+              <message name="IDS_HELLO">
+                Hello!
+              </message>
+            </if>
+            <if expr="lang == 'fr' or 'FORCE_FRENCH' in defs">
+              <message name="IDS_HELLO" internal_comment="French version">
+                Good morning
+              </message>
+            </if>
+          </messages>
+        </release>
+      </grit>'''), dir='.')
+    
+    messages_node = grd.children[0].children[0]
+    bingo_message = messages_node.children[0].children[0]
+    hello_message = messages_node.children[1].children[0]
+    french_message = messages_node.children[2].children[0]
+    assert bingo_message.name == 'message'
+    assert hello_message.name == 'message'
+    assert french_message.name == 'message'
+    
+    grd.SetOutputContext('fr', {'hello' : '1'})
+    self.failUnless(not bingo_message.SatisfiesOutputCondition())
+    self.failUnless(hello_message.SatisfiesOutputCondition())
+    self.failUnless(french_message.SatisfiesOutputCondition())
+    
+    grd.SetOutputContext('en', {'bingo' : 1})
+    self.failUnless(bingo_message.SatisfiesOutputCondition())
+    self.failUnless(not hello_message.SatisfiesOutputCondition())
+    self.failUnless(not french_message.SatisfiesOutputCondition())
+    
+    grd.SetOutputContext('en', {'FORCE_FRENCH' : '1', 'bingo' : '1'})    
+    self.failUnless(bingo_message.SatisfiesOutputCondition())
+    self.failUnless(not hello_message.SatisfiesOutputCondition())
+    self.failUnless(french_message.SatisfiesOutputCondition())
+
+
+class ReleaseNodeUnittest(unittest.TestCase):
+  def testPseudoControl(self):
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="1" source_lang_id="en-US" current_release="2" base_dir=".">
+        <release seq="1" allow_pseudo="false">
+          <messages>
+            <message name="IDS_HELLO">
+              Hello
+            </message>
+          </messages>
+          <structures>
+            <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" />
+          </structures>
+        </release>
+        <release seq="2">
+          <messages>
+            <message name="IDS_BINGO">
+              Bingo
+            </message>
+          </messages>
+          <structures>
+            <structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" />
+          </structures>
+        </release>
+      </grit>'''), util.PathFromRoot('grit/test/data'))
+    grd.RunGatherers(recursive=True)
+    
+    hello = grd.GetNodeById('IDS_HELLO')
+    aboutbox = grd.GetNodeById('IDD_ABOUTBOX')
+    bingo = grd.GetNodeById('IDS_BINGO')
+    menu = grd.GetNodeById('IDC_KLONKMENU')
+    
+    for node in [hello, aboutbox]:
+      self.failUnless(not node.PseudoIsAllowed())
+    
+    for node in [bingo, menu]:
+      self.failUnless(node.PseudoIsAllowed())
+    
+    for node in [hello, aboutbox]:
+      try:
+        formatter = node.ItemFormatter('rc_all')
+        formatter.Format(node, 'xyz-pseudo')
+        self.fail('Should have failed during Format since pseudo is not allowed')
+      except:
+        pass  # expected case
+    
+    for node in [bingo, menu]:
+      try:
+        formatter = node.ItemFormatter('rc_all')
+        formatter.Format(node, 'xyz-pseudo')
+      except:
+        self.fail('Should not have gotten exception since pseudo is allowed')
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/structure.py

@@ -0,0 +1,284 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The <structure> element.
+'''
+
+import os
+
+from grit.node import base
+from grit.node import variant
+
+from grit import constants
+from grit import exception
+from grit import util
+
+import grit.gather.rc
+import grit.gather.tr_html
+import grit.gather.admin_template
+import grit.gather.txt
+import grit.gather.muppet_strings
+
+import grit.format.rc
+import grit.format.rc_header
+
+# RTL languages
+# TODO(jennyz): remove this fixed set of RTL language array
+# when generic expand_variable code is added by grit team. 
+_RTL_LANGS = [
+  'ar',
+  'iw',
+  'ur',
+]
+
+# Type of the gatherer to use for each type attribute
+_GATHERERS = {
+  'accelerators' : grit.gather.rc.Accelerators,
+  'admin_template' : grit.gather.admin_template.AdmGatherer,
+  'dialog'  : grit.gather.rc.Dialog,
+  'menu'    : grit.gather.rc.Menu,
+  'muppet'  : grit.gather.muppet_strings.MuppetStrings,
+  'rcdata'  : grit.gather.rc.RCData,
+  'tr_html' : grit.gather.tr_html.TrHtml,
+  'txt'     : grit.gather.txt.TxtFile,
+  'version' : grit.gather.rc.Version,
+}
+
+
+# Formatter instance to use for each type attribute
+# when formatting .rc files.
+_RC_FORMATTERS = {
+  'accelerators' : grit.format.rc.RcSection(),
+  'admin_template' : grit.format.rc.RcInclude('ADM'),
+  'dialog'  : grit.format.rc.RcSection(),
+  'menu'    : grit.format.rc.RcSection(),
+  'muppet'  : grit.format.rc.RcInclude('XML'),
+  'rcdata'  : grit.format.rc.RcSection(),
+  'tr_html' : grit.format.rc.RcInclude('HTML'),
+  'txt'     : grit.format.rc.RcInclude('TXT'),
+  'version' : grit.format.rc.RcSection(),
+}
+
+
+# TODO(joi) Print a warning if the 'variant_of_revision' attribute indicates
+# that a skeleton variant is older than the original file.
+
+
+class StructureNode(base.Node):
+  '''A <structure> element.'''
+  
+  def __init__(self):
+    base.Node.__init__(self)
+    self.gatherer = None
+    self.skeletons = {}  # expressions to skeleton gatherers
+  
+  def _IsValidChild(self, child):
+    return isinstance(child, variant.SkeletonNode)
+
+  def MandatoryAttributes(self):
+    return ['type', 'name', 'file']
+
+  def DefaultAttributes(self):
+    return { 'encoding' : 'cp1252',
+             'exclude_from_rc' : 'false',
+             'line_end' : 'unix',
+             'output_encoding' : 'utf-8',
+             'generateid': 'true',
+             'expand_variables' : 'false',
+             'output_filename' : '',
+             # TODO(joi) this is a hack - should output all generated files
+             # as SCons dependencies; however, for now there is a bug I can't
+             # find where GRIT doesn't build the matching fileset, therefore
+             # this hack so that only the files you really need are marked as
+             # dependencies.
+             'sconsdep' : 'false',
+             }
+  
+  def IsExcludedFromRc(self):
+    return self.attrs['exclude_from_rc'] == 'true'
+  
+  def GetLineEnd(self):
+    '''Returns the end-of-line character or characters for files output because
+    of this node ('\r\n', '\n', or '\r' depending on the 'line_end' attribute).
+    '''
+    if self.attrs['line_end'] == 'unix':
+      return '\n'
+    elif self.attrs['line_end'] == 'windows':
+      return '\r\n'
+    elif self.attrs['line_end'] == 'mac':
+      return '\r'
+    else:
+      raise exception.UnexpectedAttribute(
+        "Attribute 'line_end' must be one of 'linux' (default), 'windows' or 'mac'")
+  
+  def GetCliques(self):
+    if self.gatherer:
+      return self.gatherer.GetCliques()
+    else:
+      return []
+  
+  def GetTextualIds(self):
+    if self.gatherer and self.attrs['type'] not in ['tr_html', 'admin_template', 'txt']:
+      return self.gatherer.GetTextualIds()
+    else:
+      return [self.attrs['name']]
+
+  def ItemFormatter(self, t):
+    if t == 'rc_header':
+      return grit.format.rc_header.Item()
+    elif (t in ['rc_all', 'rc_translateable', 'rc_nontranslateable'] and
+          self.SatisfiesOutputCondition()):
+      return _RC_FORMATTERS[self.attrs['type']]
+    else:
+      return super(type(self), self).ItemFormatter(t)
+
+  def RunGatherers(self, recursive=False, debug=False):
+    if self.gatherer:
+      return  # idempotent
+    
+    gathertype = _GATHERERS[self.attrs['type']]
+
+    if debug:
+      print 'Running gatherer %s for file %s' % (str(gathertype), self.FilenameToOpen())
+    
+    self.gatherer = gathertype.FromFile(self.FilenameToOpen(),
+                                        self.attrs['name'],
+                                        self.attrs['encoding'])
+    self.gatherer.SetUberClique(self.UberClique())
+    self.gatherer.Parse()
+    
+    for child in self.children:
+      assert isinstance(child, variant.SkeletonNode)
+      skel = gathertype.FromFile(child.FilenameToOpen(),
+                                 self.attrs['name'],
+                                 child.GetEncodingToUse())
+      skel.SetUberClique(self.UberClique())
+      skel.SetSkeleton(True)
+      skel.Parse()
+      self.skeletons[child.attrs['expr']] = skel
+      
+  def GetSkeletonGatherer(self):
+    '''Returns the gatherer for the alternate skeleton that should be used,
+    based on the expressions for selecting skeletons, or None if the skeleton
+    from the English version of the structure should be used.
+    '''
+    for expr in self.skeletons:
+      if self.EvaluateCondition(expr):
+        return self.skeletons[expr]
+    return None
+  
+  def GetFilePath(self):
+    return self.ToRealPath(self.attrs['file'])
+  
+  def HasFileForLanguage(self):
+    return self.attrs['type'] in ['tr_html', 'admin_template', 'txt', 'muppet']
+  
+  def FileForLanguage(self, lang, output_dir, create_file=True,
+                      return_if_not_generated=True):
+    '''Returns the filename of the file associated with this structure,
+    for the specified language.
+    
+    Args:
+      lang: 'fr'
+      output_dir: 'c:\temp'
+      create_file: True
+    '''
+    assert self.HasFileForLanguage()
+    if (lang == self.GetRoot().GetSourceLanguage() and 
+        self.attrs['expand_variables'] != 'true'):
+      if return_if_not_generated:
+        return self.GetFilePath()
+      else:
+        return None
+    else:
+      if self.attrs['output_filename'] != '':
+        filename = self.attrs['output_filename']
+      else:
+        filename = os.path.basename(self.attrs['file'])
+      assert len(filename)
+      filename = '%s_%s' % (lang, filename)
+      filename = os.path.join(output_dir, filename)
+      
+      if create_file:
+        text = self.gatherer.Translate(
+          lang,
+          pseudo_if_not_available=self.PseudoIsAllowed(),
+          fallback_to_english=self.ShouldFallbackToEnglish(),
+          skeleton_gatherer=self.GetSkeletonGatherer())
+        
+        file_object = util.WrapOutputStream(file(filename, 'wb'),
+                                            self._GetOutputEncoding())
+        file_contents = util.FixLineEnd(text, self.GetLineEnd())
+        if self.attrs['expand_variables'] == 'true':
+          file_contents = file_contents.replace('[GRITLANGCODE]', lang)
+          # TODO(jennyz): remove this hard coded logic for expanding
+          # [GRITDIR] variable for RTL languages when the generic
+          # expand_variable code is added by grit team.
+          if lang in _RTL_LANGS :
+            file_contents = file_contents.replace('[GRITDIR]', 'dir="RTL"')
+          else :
+            file_contents = file_contents.replace('[GRITDIR]', 'dir="LTR"')
+        if self._ShouldAddBom():
+          file_object.write(constants.BOM)
+        file_object.write(file_contents)
+        file_object.close()
+      
+      return filename
+  
+  def _GetOutputEncoding(self):
+    '''Python doesn't natively support UTF encodings with a BOM signature,
+    so we add support by allowing you to append '-sig' to the encoding name.
+    This function returns the specified output encoding minus that part.
+    '''
+    enc = self.attrs['output_encoding']
+    if enc.endswith('-sig'):
+      return enc[0:len(enc) - len('-sig')]
+    else:
+      return enc
+  
+  def _ShouldAddBom(self):
+    '''Returns true if output files should have the Unicode BOM prepended.
+    '''
+    return self.attrs['output_encoding'].endswith('-sig')
+  
+  # static method
+  def Construct(parent, name, type, file, encoding='cp1252'):
+    '''Creates a new node which is a child of 'parent', with attributes set
+    by parameters of the same name.
+    '''
+    node = StructureNode()
+    node.StartParsing('structure', parent)
+    node.HandleAttribute('name', name)
+    node.HandleAttribute('type', type)
+    node.HandleAttribute('file', file)
+    node.HandleAttribute('encoding', encoding)
+    node.EndParsing()
+    return node
+  Construct = staticmethod(Construct)

tools/grit/grit/node/structure_unittest.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for <structure> nodes.
+'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import unittest
+import StringIO
+
+from grit.node import structure
+from grit import grd_reader
+from grit import util
+
+
+class StructureUnittest(unittest.TestCase):
+  def testSkeleton(self):
+    grd = grd_reader.Parse(StringIO.StringIO(
+      '''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <release seq="3">
+          <structures>
+            <structure type="dialog" name="IDD_ABOUTBOX" file="klonk.rc" encoding="utf-16-le">
+              <skeleton expr="lang == 'fr'" variant_of_revision="1" file="klonk-alternate-skeleton.rc" />
+            </structure>
+          </structures>
+        </release>
+      </grit>'''), dir=util.PathFromRoot('grit\\test\\data'))
+    grd.RunGatherers(recursive=True)
+    grd.output_language = 'fr'
+    
+    node = grd.GetNodeById('IDD_ABOUTBOX')
+    formatter = node.ItemFormatter('rc_all')
+    self.failUnless(formatter)
+    transl = formatter.Format(node, 'fr')
+    
+    self.failUnless(transl.count('040704') and transl.count('110978'))
+    self.failUnless(transl.count('2005",IDC_STATIC'))
+  
+  def testOutputEncoding(self):
+    grd = grd_reader.Parse(StringIO.StringIO(
+      '''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <release seq="3">
+          <structures>
+            <structure type="dialog" name="IDD_ABOUTBOX" file="klonk.rc" encoding="utf-16-le" output_encoding="utf-8-sig" />
+          </structures>
+        </release>
+      </grit>'''), dir=util.PathFromRoot('grit\\test\\data'))
+    node = grd.GetNodeById('IDD_ABOUTBOX')
+    self.failUnless(node._GetOutputEncoding() == 'utf-8')
+    self.failUnless(node._ShouldAddBom())
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/node/variant.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The <skeleton> element.
+'''
+
+
+from grit.node import base
+
+
+class SkeletonNode(base.Node):
+  '''A <skeleton> element.'''
+  
+  # TODO(joi) Support inline skeleton variants as CDATA instead of requiring
+  # a 'file' attribute.
+  
+  def MandatoryAttributes(self):
+    return ['expr', 'variant_of_revision', 'file']
+  
+  def DefaultAttributes(self):
+    '''If not specified, 'encoding' will actually default to the parent node's
+    encoding.
+    '''
+    return {'encoding' : ''}
+
+  def _ContentType(self):
+    if self.attrs.has_key('file'):
+      return self._CONTENT_TYPE_NONE
+    else:
+      return self._CONTENT_TYPE_CDATA
+  
+  def GetEncodingToUse(self):
+    if self.attrs['encoding'] == '':
+      return self.parent.attrs['encoding']
+    else:
+      return self.attrs['encoding']
+
+  def GetFilePath(self):
+    return self.ToRealPath(self.attrs['file'])

tools/grit/grit/pseudo.py

@@ -0,0 +1,154 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Pseudotranslation support.  Our pseudotranslations are based on the
+P-language, which is a simple vowel-extending language.  Examples of P:
+  - "hello" becomes "hepellopo"
+  - "howdie" becomes "hopowdiepie"
+  - "because" becomes "bepecaupause" (but in our implementation we don't
+    handle the silent e at the end so it actually would return "bepecaupausepe"
+
+The P-language has the excellent quality of increasing the length of text
+by around 30-50% which is great for pseudotranslations, to stress test any
+GUI layouts etc.
+
+To make the pseudotranslations more obviously "not a translation" and to make
+them exercise any code that deals with encodings, we also transform all English
+vowels into equivalent vowels with diacriticals on them (rings, acutes,
+diaresis, and circumflex), and we write the "p" in the P-language as a Hebrew
+character Qof.  It looks sort of like a latin character "p" but it is outside
+the latin-1 character set which will stress character encoding bugs.
+'''
+
+import re
+import types
+
+from grit import tclib
+
+
+# An RFC language code for the P pseudolanguage.
+PSEUDO_LANG = 'x-P-pseudo'
+
+# Hebrew character Qof.  It looks kind of like a 'p' but is outside
+# the latin-1 character set which is good for our purposes.
+# TODO(joi) For now using P instead of Qof, because of some bugs it used.  Find
+# a better solution, i.e. one that introduces a non-latin1 character into the
+# pseudotranslation.
+#_QOF = u'\u05e7'
+_QOF = u'P'
+
+# How we map each vowel.
+_VOWELS = {
+  u'a' : u'\u00e5',  # a with ring
+  u'e' : u'\u00e9',  # e acute
+  u'i' : u'\u00ef',  # i diaresis
+  u'o' : u'\u00f4',  # o circumflex
+  u'u' : u'\u00fc',  # u diaresis
+  u'y' : u'\u00fd',  # y acute
+  u'A' : u'\u00c5',  # A with ring
+  u'E' : u'\u00c9',  # E acute
+  u'I' : u'\u00cf',  # I diaresis
+  u'O' : u'\u00d4',  # O circumflex
+  u'U' : u'\u00dc',  # U diaresis
+  u'Y' : u'\u00dd',  # Y acute
+}
+
+# Matches vowels and P
+_PSUB_RE = re.compile("(%s)" % '|'.join(_VOWELS.keys() + ['P']))
+
+
+# Pseudotranslations previously created.  This is important for performance
+# reasons, especially since we routinely pseudotranslate the whole project
+# several or many different times for each build.
+_existing_translations = {}
+
+
+def MapVowels(str, also_p = False):
+  '''Returns a copy of 'str' where characters that exist as keys in _VOWELS
+  have been replaced with the corresponding value.  If also_p is true, this
+  function will also change capital P characters into a Hebrew character Qof.
+  '''
+  def Repl(match):
+    if match.group() == 'p':
+      if also_p:
+        return _QOF
+      else:
+        return 'p'
+    else:
+      return _VOWELS[match.group()]
+  return _PSUB_RE.sub(Repl, str)
+
+
+def PseudoString(str):
+  '''Returns a pseudotranslation of the provided string, in our enhanced
+  P-language.'''
+  if str in _existing_translations:
+    return _existing_translations[str]
+  
+  outstr = u''
+  ix = 0
+  while ix < len(str):
+    if str[ix] not in _VOWELS.keys():
+      outstr += str[ix]
+      ix += 1
+    else:
+      # We want to treat consecutive vowels as one composite vowel.  This is not
+      # always accurate e.g. in composite words but good enough.
+      consecutive_vowels = u''
+      while ix < len(str) and str[ix] in _VOWELS.keys():
+        consecutive_vowels += str[ix]
+        ix += 1
+      changed_vowels = MapVowels(consecutive_vowels)
+      outstr += changed_vowels
+      outstr += _QOF
+      outstr += changed_vowels
+  
+  _existing_translations[str] = outstr
+  return outstr
+
+
+def PseudoMessage(message):
+  '''Returns a pseudotranslation of the provided message.
+  
+  Args:
+    message: tclib.Message()
+  
+  Return:
+    tclib.Translation()
+  '''
+  transl = tclib.Translation()
+  
+  for part in message.GetContent():
+    if isinstance(part, tclib.Placeholder):
+      transl.AppendPlaceholder(part)
+    else:
+      transl.AppendText(PseudoString(part))
+  
+  return transl

tools/grit/grit/pseudo_unittest.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.pseudo'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+
+from grit import pseudo
+from grit import tclib
+
+
+class PseudoUnittest(unittest.TestCase):
+  def testVowelMapping(self):
+    self.failUnless(pseudo.MapVowels('abebibobuby') ==
+                    u'\u00e5b\u00e9b\u00efb\u00f4b\u00fcb\u00fd')
+    self.failUnless(pseudo.MapVowels('ABEBIBOBUBY') ==
+                    u'\u00c5B\u00c9B\u00cfB\u00d4B\u00dcB\u00dd')
+  
+  def testPseudoString(self):
+    out = pseudo.PseudoString('hello')
+    self.failUnless(out == pseudo.MapVowels(u'hePelloPo', True))
+  
+  def testConsecutiveVowels(self):
+    out = pseudo.PseudoString("beautiful weather, ain't it?")
+    self.failUnless(out == pseudo.MapVowels(
+      u"beauPeautiPifuPul weaPeathePer, aiPain't iPit?", 1))
+  
+  def testCapitals(self):
+    out = pseudo.PseudoString("HOWDIE DOODIE, DR. JONES")
+    self.failUnless(out == pseudo.MapVowels(
+      u"HOPOWDIEPIE DOOPOODIEPIE, DR. JOPONEPES", 1))
+
+  def testPseudoMessage(self):
+    msg = tclib.Message(text='Hello USERNAME, how are you?',
+                        placeholders=[
+                          tclib.Placeholder('USERNAME', '%s', 'Joi')])
+    trans = pseudo.PseudoMessage(msg)
+    # TODO(joi) It would be nicer if 'you' -> 'youPou' instead of
+    # 'you' -> 'youPyou' and if we handled the silent e in 'are'
+    self.failUnless(trans.GetPresentableContent() ==
+                    pseudo.MapVowels(
+                      u'HePelloPo USERNAME, hoPow aParePe youPyou?', 1))
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/scons.py

@@ -0,0 +1,167 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''SCons integration for GRIT.
+'''
+
+# NOTE:  DO NOT IMPORT ANY GRIT STUFF HERE - we import lazily so that grit and
+# its dependencies aren't imported until actually needed.
+
+import os
+import types
+
+def _IsDebugEnabled():
+  return 'GRIT_DEBUG' in os.environ and os.environ['GRIT_DEBUG'] == '1'
+
+def _SourceToFile(source):
+  '''Return the path to the source file, given the 'source' argument as provided
+  by SCons to the _Builder or _Emitter functions.
+  '''
+  # Get the filename of the source.  The 'source' parameter can be a string,
+  # a "node", or a list of strings or nodes.
+  if isinstance(source, types.ListType):
+    source = str(source[0])
+  else:
+    source = str(source)
+  return source
+
+
+def _Builder(target, source, env):
+  from grit import grit_runner
+  from grit.tool import build
+  options = grit_runner.Options()
+  # This sets options to default values  TODO(joi) Remove verbose
+  options.ReadOptions(['-v'])
+  options.input = _SourceToFile(source)
+  
+  # TODO(joi) Check if we can get the 'verbose' option from the environment.  
+  
+  builder = build.RcBuilder()
+  
+  # Get the CPP defines from the environment.
+  for flag in env['RCFLAGS']:
+    if flag.startswith('/D'):
+      flag = flag[2:]
+    name, val = build.ParseDefine(flag)
+    # Only apply to first instance of a given define
+    if name not in builder.defines:
+      builder.defines[name] = val
+  
+  # To ensure that our output files match what we promised SCons, we
+  # use the list of targets provided by SCons and update the file paths in
+  # our .grd input file with the targets.
+  builder.scons_targets = [str(t) for t in target]
+  builder.Run(options, [])
+  return None  # success
+
+
+def _Emitter(target, source, env):
+  '''A SCons emitter for .grd files, which modifies the list of targes to
+  include all files in the <outputs> section of the .grd file as well as
+  any other files output by 'grit build' for the .grd file.
+  '''
+  from grit import util
+  from grit import grd_reader
+  
+  base_dir = util.dirname(str(target[0]))
+  
+  grd = grd_reader.Parse(_SourceToFile(source), debug=_IsDebugEnabled())
+  
+  target = []
+  lang_folders = {}
+  # Add all explicitly-specified output files
+  for output in grd.GetOutputFiles():
+    path = os.path.join(base_dir, output.GetFilename())
+    target.append(path)
+    if _IsDebugEnabled():
+      print "GRIT: Added target %s" % path
+    if output.attrs['lang'] != '':
+      lang_folders[output.attrs['lang']] = os.path.dirname(path)
+  
+  # Add all generated files, once for each output language.
+  for node in grd:
+    if node.name == 'structure':
+      # TODO(joi) Should remove the "if sconsdep is true" thing as it is a
+      # hack - see grit/node/structure.py
+      if node.HasFileForLanguage() and node.attrs['sconsdep'] == 'true':
+        for lang in lang_folders:
+          path = node.FileForLanguage(lang, lang_folders[lang],
+                                      create_file=False,
+                                      return_if_not_generated=False)
+          if path:
+            target.append(path)
+            if _IsDebugEnabled():
+              print "GRIT: Added target %s" % path
+  
+  # return target and source lists
+  return (target, source)
+
+
+def _Scanner(file_node, env, path):
+  '''A SCons scanner function for .grd files, which outputs the list of files
+  that changes in could change the output of building the .grd file.
+  '''
+  from grit import grd_reader
+  
+  grd = grd_reader.Parse(str(file_node), debug=_IsDebugEnabled())
+  files = []
+  for node in grd:
+    if (node.name == 'structure' or node.name == 'skeleton' or
+        (node.name == 'file' and node.parent and
+         node.parent.name == 'translations')):
+      files.append(os.path.abspath(node.GetFilePath()))
+  return files
+
+
+# Function name is mandated by newer versions of SCons.
+def generate(env):
+  # Importing this module should be possible whenever this function is invoked
+  # since it should only be invoked by SCons.
+  import SCons.Builder
+  import SCons.Action
+  
+  # The varlist parameter tells SCons that GRIT needs to be invoked again
+  # if RCFLAGS has changed since last compilation.
+  action = SCons.Action.FunctionAction(_Builder, varlist=['RCFLAGS'])
+  
+  builder = SCons.Builder.Builder(action=action,
+                              emitter=_Emitter,
+                              src_suffix='.grd')
+  
+  scanner = env.Scanner(function=_Scanner, name='GRIT', skeys=['.grd'])
+  
+  # add our builder and scanner to the environment
+  env.Append(BUILDERS = {'GRIT': builder})
+  env.Prepend(SCANNERS = scanner)
+
+
+# Function name is mandated by newer versions of SCons.
+def exists(env):
+  return 1

tools/grit/grit/shortcuts.py

@@ -0,0 +1,119 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Stuff to prevent conflicting shortcuts.
+'''
+
+import re
+
+from grit import util
+
+
+class ShortcutGroup(object):
+  '''Manages a list of cliques that belong together in a single shortcut
+  group.  Knows how to detect conflicting shortcut keys.
+  '''
+
+  # Matches shortcut keys, e.g. &J
+  SHORTCUT_RE = re.compile('([^&]|^)(&[A-Za-z])')
+
+  def __init__(self, name):
+    self.name = name
+    # Map of language codes to shortcut keys used (which is a map of
+    # shortcut keys to counts).
+    self.keys_by_lang = {}
+    # List of cliques in this group
+    self.cliques = []
+  
+  def AddClique(self, c):
+    for existing_clique in self.cliques:
+      if existing_clique.GetId() == c.GetId():
+        # This happens e.g. when we have e.g.
+        # <if expr1><structure 1></if> <if expr2><structure 2></if>
+        # where only one will really be included in the output.
+        return
+    
+    self.cliques.append(c)
+    for (lang, msg) in c.clique.items():
+      if lang not in self.keys_by_lang:
+        self.keys_by_lang[lang] = {}
+      keymap = self.keys_by_lang[lang]
+      
+      content = msg.GetRealContent()
+      keys = [groups[1] for groups in self.SHORTCUT_RE.findall(content)]
+      for key in keys:
+        key = key.upper()
+        if key in keymap:
+          keymap[key] += 1
+        else:
+          keymap[key] = 1
+  
+  def GenerateWarnings(self, tc_project):
+    # For any language that has more than one occurrence of any shortcut,
+    # make a list of the conflicting shortcuts.
+    problem_langs = {}
+    for (lang, keys) in self.keys_by_lang.items():
+      for (key, count) in keys.items():
+        if count > 1:
+          if lang not in problem_langs:
+            problem_langs[lang] = []
+          problem_langs[lang].append(key)
+    
+    warnings = []
+    if len(problem_langs):
+      warnings.append("WARNING - duplicate keys exist in shortcut group %s" %
+                      self.name)
+      for (lang,keys) in problem_langs.items():
+        warnings.append("  %6s duplicates: %s" % (lang, ', '.join(keys)))
+    return warnings
+
+
+def GenerateDuplicateShortcutsWarnings(uberclique, tc_project):
+  '''Given an UberClique and a project name, will print out helpful warnings
+  if there are conflicting shortcuts within shortcut groups in the provided
+  UberClique.
+  
+  Args:
+    uberclique: clique.UberClique()
+    tc_project: 'MyProjectNameInTheTranslationConsole'
+  
+  Returns:
+    ['warning line 1', 'warning line 2', ...]
+  '''
+  warnings = []
+  groups = {}
+  for c in uberclique.AllCliques():
+    for group in c.shortcut_groups:
+      if group not in groups:
+        groups[group] = ShortcutGroup(group)
+      groups[group].AddClique(c)
+  for group in groups.values():
+    warnings += group.GenerateWarnings(tc_project)  
+  return warnings

tools/grit/grit/shortcuts_unittests.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.shortcuts
+'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+
+from grit import shortcuts
+from grit import clique
+from grit import tclib
+from grit.gather import rc
+
+class ShortcutsUnittest(unittest.TestCase):
+  
+  def setUp(self):
+    self.uq = clique.UberClique()
+  
+  def testFunctionality(self):
+    c = self.uq.MakeClique(tclib.Message(text="Hello &there"))
+    c.AddToShortcutGroup('group_name')
+    c = self.uq.MakeClique(tclib.Message(text="Howdie &there partner"))
+    c.AddToShortcutGroup('group_name')
+    
+    warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT')
+    self.failUnless(warnings)
+  
+  def testAmpersandEscaping(self):
+    c = self.uq.MakeClique(tclib.Message(text="Hello &there"))
+    c.AddToShortcutGroup('group_name')
+    c = self.uq.MakeClique(tclib.Message(text="S&&T are the &letters S and T"))
+    c.AddToShortcutGroup('group_name')
+    
+    warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT')
+    self.failUnless(len(warnings) == 0)
+  
+  def testDialog(self):
+    dlg = rc.Dialog('''\
+IDD_SIDEBAR_RSS_PANEL_PROPPAGE DIALOGEX 0, 0, 239, 221
+STYLE DS_SETFONT | DS_FIXEDSYS | WS_CHILD
+FONT 8, "MS Shell Dlg", 400, 0, 0x1
+BEGIN
+    PUSHBUTTON      "Add &URL",IDC_SIDEBAR_RSS_ADD_URL,182,53,57,14
+    EDITTEXT        IDC_SIDEBAR_RSS_NEW_URL,0,53,178,15,ES_AUTOHSCROLL
+    PUSHBUTTON      "&Remove",IDC_SIDEBAR_RSS_REMOVE,183,200,56,14
+    PUSHBUTTON      "&Edit",IDC_SIDEBAR_RSS_EDIT,123,200,56,14
+    CONTROL         "&Automatically add commonly viewed clips",
+                    IDC_SIDEBAR_RSS_AUTO_ADD,"Button",BS_AUTOCHECKBOX | 
+                    BS_MULTILINE | WS_TABSTOP,0,200,120,17
+    PUSHBUTTON      "",IDC_SIDEBAR_RSS_HIDDEN,179,208,6,6,NOT WS_VISIBLE
+    LTEXT           "You can display clips from blogs, news sites, and other online sources.",
+                    IDC_STATIC,0,0,239,10
+    LISTBOX         IDC_SIDEBAR_DISPLAYED_FEED_LIST,0,69,239,127,LBS_SORT | 
+                    LBS_OWNERDRAWFIXED | LBS_HASSTRINGS | 
+                    LBS_NOINTEGRALHEIGHT | WS_VSCROLL | WS_HSCROLL | 
+                    WS_TABSTOP
+    LTEXT           "Add a clip from a recently viewed website by clicking Add Recent Clips.",
+                    IDC_STATIC,0,13,141,19
+    LTEXT           "Or, if you know a site supports RSS or Atom, you can enter the RSS or Atom URL below and add it to your list of Web Clips.",
+                    IDC_STATIC,0,33,239,18
+    PUSHBUTTON      "Add Recent &Clips (10)...",
+                    IDC_SIDEBAR_RSS_ADD_RECENT_CLIPS,146,14,93,14
+END''')
+    dlg.SetUberClique(self.uq)
+    dlg.Parse()
+    
+    warnings = shortcuts.GenerateDuplicateShortcutsWarnings(self.uq, 'PROJECT')
+    self.failUnless(len(warnings) == 0)
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/tclib.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Adaptation of the extern.tclib classes for our needs.
+'''
+
+
+import re
+import types
+
+from grit import exception
+import grit.extern.tclib
+
+def Identity(i):
+  return i
+
+
+class BaseMessage(object):
+  '''Base class with methods shared by Message and Translation.
+  '''
+  
+  def __init__(self, text='', placeholders=[], description='', meaning=''):
+    self.parts = []
+    self.placeholders = []
+    self.description = description
+    self.meaning = meaning
+    self.dirty = True  # True if self.id is (or might be) wrong
+    self.id = 0
+    
+    if text != '':
+      if not placeholders or placeholders == []:
+        self.AppendText(text)
+      else:
+        tag_map = {}
+        for placeholder in placeholders:
+          tag_map[placeholder.GetPresentation()] = [placeholder, 0]
+        tag_re = '(' + '|'.join(tag_map.keys()) + ')'
+        # This creates a regexp like '(TAG1|TAG2|TAG3)'
+        chunked_text = re.split(tag_re, text)
+        for chunk in chunked_text:
+          if chunk: # ignore empty chunk
+            if tag_map.has_key(chunk):
+              self.AppendPlaceholder(tag_map[chunk][0])
+              tag_map[chunk][1] += 1 # increase placeholder use count
+            else:
+              self.AppendText(chunk)
+        for key in tag_map.keys():
+          assert tag_map[key][1] != 0
+  
+  def GetRealContent(self, escaping_function=Identity):
+    '''Returns the original content, i.e. what your application and users
+    will see.
+    
+    Specify a function to escape each translateable bit, if you like.
+    '''
+    bits = []
+    for item in self.parts:
+      if isinstance(item, types.StringTypes):
+        bits.append(escaping_function(item))
+      else:
+        bits.append(item.GetOriginal())
+    return ''.join(bits)
+  
+  def GetPresentableContent(self):
+    presentable_content = []
+    for part in self.parts:
+      if isinstance(part, Placeholder):
+        presentable_content.append(part.GetPresentation())
+      else:
+        presentable_content.append(part)
+    return ''.join(presentable_content)
+  
+  def AppendPlaceholder(self, placeholder):
+    assert isinstance(placeholder, Placeholder)
+    dup = False
+    for other in self.GetPlaceholders():
+      if (other.presentation.find(placeholder.presentation) != -1 or
+          placeholder.presentation.find(other.presentation) != -1):
+        assert(False, "Placeholder names must be unique and must not overlap")
+      if other.presentation == placeholder.presentation:
+        assert other.original == placeholder.original
+        dup = True
+    
+    if not dup:
+      self.placeholders.append(placeholder)
+    self.parts.append(placeholder)
+    self.dirty = True
+  
+  def AppendText(self, text):
+    assert isinstance(text, types.StringTypes)
+    assert text != ''
+    
+    self.parts.append(text)
+    self.dirty = True
+  
+  def GetContent(self):
+    '''Returns the parts of the message.  You may modify parts if you wish.
+    Note that you must not call GetId() on this object until you have finished
+    modifying the contents.
+    '''
+    self.dirty = True  # user might modify content
+    return self.parts
+  
+  def GetDescription(self):
+    return self.description
+  
+  def SetDescription(self, description):
+    self.description = description
+  
+  def GetMeaning(self):
+    return self.meaning
+  
+  def GetId(self):
+    if self.dirty:
+      self.id = self.GenerateId()
+      self.dirty = False
+    return self.id
+  
+  def GenerateId(self):
+    # Must use a UTF-8 encoded version of the presentable content, along with
+    # the meaning attribute, to match the TC.
+    return grit.extern.tclib.GenerateMessageId(
+      self.GetPresentableContent().encode('utf-8'), self.meaning)
+  
+  def GetPlaceholders(self):
+    return self.placeholders
+  
+  def FillTclibBaseMessage(self, msg):
+    msg.SetDescription(self.description.encode('utf-8'))
+    
+    for part in self.parts:
+      if isinstance(part, Placeholder):
+        ph = grit.extern.tclib.Placeholder(
+          part.presentation.encode('utf-8'),
+          part.original.encode('utf-8'),
+          part.example.encode('utf-8'))
+        msg.AppendPlaceholder(ph)
+      else:
+        msg.AppendText(part.encode('utf-8'))
+
+
+class Message(BaseMessage):
+  '''A message.'''  
+  
+  def __init__(self, text='', placeholders=[], description='', meaning=''):
+    BaseMessage.__init__(self, text, placeholders, description, meaning)
+  
+  def ToTclibMessage(self):
+    msg = grit.extern.tclib.Message('utf-8', meaning=self.meaning)
+    self.FillTclibBaseMessage(msg)
+    return msg
+
+class Translation(BaseMessage):
+  '''A translation.'''
+  
+  def __init__(self, text='', id='', placeholders=[], description='', meaning=''):
+    BaseMessage.__init__(self, text, placeholders, description, meaning)
+    self.id = id
+  
+  def GetId(self):
+    assert id != '', "ID has not been set."
+    return self.id
+  
+  def SetId(self, id):
+    self.id = id
+  
+  def ToTclibMessage(self):
+    msg = grit.extern.tclib.Message(
+      'utf-8', id=self.id, meaning=self.meaning)
+    self.FillTclibBaseMessage(msg)
+    return msg
+
+
+class Placeholder(grit.extern.tclib.Placeholder):
+  '''Modifies constructor to accept a Unicode string
+  '''
+  
+  # Must match placeholder presentation names
+  _NAME_RE = re.compile('[A-Za-z0-9_]+')
+  
+  def __init__(self, presentation, original, example):
+    '''Creates a new placeholder.
+    
+    Args:
+      presentation: 'USERNAME'
+      original: '%s'
+      example: 'Joi'
+    '''
+    assert presentation != ''
+    assert original != ''
+    assert example != ''
+    if not self._NAME_RE.match(presentation):
+      raise exception.InvalidPlaceholderName(presentation)
+    self.presentation = presentation
+    self.original = original
+    self.example = example
+  
+  def GetPresentation(self):
+    return self.presentation
+  
+  def GetOriginal(self):
+    return self.original
+  
+  def GetExample(self):
+    return self.example
+

tools/grit/grit/tclib_unittest.py

@@ -0,0 +1,189 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.tclib'''
+
+
+import sys
+import os.path
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import types
+import unittest
+
+from grit import tclib
+
+import grit.extern.tclib
+
+
+class TclibUnittest(unittest.TestCase):
+  def testInit(self):
+    msg = tclib.Message(text=u'Hello Earthlings')
+    self.failUnless(msg.GetPresentableContent() == 'Hello Earthlings')
+    self.failUnless(isinstance(msg.GetPresentableContent(), types.StringTypes))
+  
+  def testGetAttr(self):
+    msg = tclib.Message()
+    msg.AppendText(u'Hello')  # Tests __getattr__
+    self.failUnless(msg.GetPresentableContent() == 'Hello')
+    self.failUnless(isinstance(msg.GetPresentableContent(), types.StringTypes))
+  
+  def testAll(self):
+    text = u'Howdie USERNAME'
+    phs = [tclib.Placeholder(u'USERNAME', u'%s', 'Joi')]
+    msg = tclib.Message(text=text, placeholders=phs)
+    self.failUnless(msg.GetPresentableContent() == 'Howdie USERNAME')
+    
+    trans = tclib.Translation(text=text, placeholders=phs)
+    self.failUnless(trans.GetPresentableContent() == 'Howdie USERNAME')
+    self.failUnless(isinstance(trans.GetPresentableContent(), types.StringTypes))
+
+  def testUnicodeReturn(self):
+    text = u'\u00fe'
+    msg = tclib.Message(text=text)
+    self.failUnless(msg.GetPresentableContent() == text)
+    from_list = msg.GetContent()[0]
+    self.failUnless(from_list == text)
+    
+  def testRegressionTranslationInherited(self):
+    '''Regression tests a bug that was caused by grit.tclib.Translation
+    inheriting from the translation console's Translation object
+    instead of only owning an instance of it.
+    '''
+    msg = tclib.Message(text=u"BLA1\r\nFrom: BLA2 \u00fe BLA3",
+                        placeholders=[
+                          tclib.Placeholder('BLA1', '%s', '%s'),
+                          tclib.Placeholder('BLA2', '%s', '%s'),
+                          tclib.Placeholder('BLA3', '%s', '%s')])
+    transl = tclib.Translation(text=msg.GetPresentableContent(),
+                               placeholders=msg.GetPlaceholders())
+    content = transl.GetContent()
+    self.failUnless(isinstance(content[3], types.UnicodeType))
+  
+  def testFingerprint(self):
+    # This has Windows line endings.  That is on purpose.
+    id = grit.extern.tclib.GenerateMessageId(
+      'Google Desktop for Enterprise\r\n'
+      'Copyright (C) 2006 Google Inc.\r\n'
+      'All Rights Reserved\r\n'
+      '\r\n'
+      '---------\r\n'
+      'Contents\r\n'
+      '---------\r\n'
+      'This distribution contains the following files:\r\n'
+      '\r\n'
+      'GoogleDesktopSetup.msi - Installation and setup program\r\n'
+      'GoogleDesktop.adm - Group Policy administrative template file\r\n'
+      'AdminGuide.pdf - Google Desktop for Enterprise administrative guide\r\n'
+      '\r\n'
+      '\r\n'
+      '--------------\r\n'
+      'Documentation\r\n'
+      '--------------\r\n'
+      'Full documentation and installation instructions are in the \r\n'
+      'administrative guide, and also online at \r\n'
+      'http://desktop.google.com/enterprise/adminguide.html.\r\n'
+      '\r\n'
+      '\r\n'
+      '------------------------\r\n'
+      'IBM Lotus Notes Plug-In\r\n'
+      '------------------------\r\n'
+      'The Lotus Notes plug-in is included in the release of Google \r\n'
+      'Desktop for Enterprise. The IBM Lotus Notes Plug-in for Google \r\n'
+      'Desktop indexes mail, calendar, task, contact and journal \r\n'
+      'documents from Notes.  Discussion documents including those from \r\n'
+      'the discussion and team room templates can also be indexed by \r\n'
+      'selecting an option from the preferences.  Once indexed, this data\r\n'
+      'will be returned in Google Desktop searches.  The corresponding\r\n'
+      'document can be opened in Lotus Notes from the Google Desktop \r\n'
+      'results page.\r\n'
+      '\r\n'
+      'Install: The plug-in will install automatically during the Google \r\n'
+      'Desktop setup process if Lotus Notes is already installed.  Lotus \r\n'
+      'Notes must not be running in order for the install to occur.  \r\n'
+      '\r\n'
+      'Preferences: Preferences and selection of databases to index are\r\n'
+      'set in the \'Google Desktop for Notes\' dialog reached through the \r\n'
+      '\'Actions\' menu.\r\n'
+      '\r\n'
+      'Reindexing: Selecting \'Reindex all databases\' will index all the \r\n'
+      'documents in each database again.\r\n'
+      '\r\n'
+      '\r\n'
+      'Notes Plug-in Known Issues\r\n'
+      '---------------------------\r\n'
+      '\r\n'
+      'If the \'Google Desktop for Notes\' item is not available from the \r\n'
+      'Lotus Notes Actions menu, then installation was not successful. \r\n'
+      'Installation consists of writing one file, notesgdsplugin.dll, to \r\n'
+      'the Notes application directory and a setting to the notes.ini \r\n'
+      'configuration file.  The most likely cause of an unsuccessful \r\n'
+      'installation is that the installer was not able to locate the \r\n'
+      'notes.ini file. Installation will complete if the user closes Notes\r\n'
+      'and manually adds the following setting to this file on a new line:\r\n'
+      'AddinMenus=notegdsplugin.dll\r\n'
+      '\r\n'
+      'If the notesgdsplugin.dll file is not in the application directory\r\n'
+      '(e.g., C:\Program Files\Lotus\Notes) after Google Desktop \r\n'
+      'installation, it is likely that Notes was not installed correctly. \r\n'
+      '\r\n'
+      'Only local databases can be indexed.  If they can be determined, \r\n'
+      'the user\'s local mail file and address book will be included in the\r\n'
+      'list automatically.  Mail archives and other databases must be \r\n'
+      'added with the \'Add\' button.\r\n'
+      '\r\n'
+      'Some users may experience performance issues during the initial \r\n'
+      'indexing of a database.  The \'Perform the initial index of a \r\n'
+      'database only when I\'m idle\' option will limit the indexing process\r\n'
+      'to times when the user is not using the machine. If this does not \r\n'
+      'alleviate the problem or the user would like to continually index \r\n'
+      'but just do so more slowly or quickly, the GoogleWaitTime notes.ini\r\n'
+      'value can be set. Increasing the GoogleWaitTime value will slow \r\n'
+      'down the indexing process, and lowering the value will speed it up.\r\n'
+      'A value of zero causes the fastest possible indexing.  Removing the\r\n'
+      'ini parameter altogether returns it to the default (20).\r\n'
+      '\r\n'
+      'Crashes have been known to occur with certain types of history \r\n'
+      'bookmarks.  If the Notes client seems to crash randomly, try \r\n'
+      'disabling the \'Index note history\' option.  If it crashes before,\r\n'
+      'you can get to the preferences, add the following line to your \r\n'
+      'notes.ini file:\r\n'
+      'GDSNoIndexHistory=1\r\n')
+    self.failUnless(id == '8961534701379422820')
+
+  def testPlaceholderNameChecking(self):
+    try:
+      ph = tclib.Placeholder('BINGO BONGO', 'bla', 'bla')
+    except exception.InvalidPlaceholderName:
+      pass  # Expect exception to be thrown because presentation contained space
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/test_suite_all.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit test suite that collects all test cases for GRIT.'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+
+
+# TODO(joi) Use unittest.defaultTestLoader to automatically load tests
+# from modules. Iterating over the directory and importing could then
+# automate this all the way, if desired.
+
+
+class TestSuiteAll(unittest.TestSuite):
+  def __init__(self):
+    super(type(self), self).__init__()
+    # Imports placed here to prevent circular imports.
+    from grit import grd_reader_unittest
+    from grit import grit_runner_unittest
+    from grit.node import base_unittest
+    from grit.node import io_unittest
+    from grit import clique_unittest
+    from grit.node import misc_unittest
+    from grit.gather import rc_unittest
+    from grit.gather import tr_html_unittest
+    from grit.node import message_unittest
+    from grit import tclib_unittest
+    import grit.format.rc_unittest
+    from grit.tool import rc2grd_unittest
+    from grit.tool import transl2tc_unittest
+    from grit.gather import txt_unittest
+    from grit.gather import admin_template_unittest
+    from grit import xtb_reader_unittest
+    from grit import util_unittest
+    from grit.tool import preprocess_unittest
+    from grit.tool import postprocess_unittest
+    from grit import shortcuts_unittests
+    from grit.gather import muppet_strings_unittest
+    from grit.node.custom import filename_unittest
+    
+    test_classes = [
+      base_unittest.NodeUnittest,
+      io_unittest.FileNodeUnittest,
+      grit_runner_unittest.OptionArgsUnittest,
+      grd_reader_unittest.GrdReaderUnittest,
+      clique_unittest.MessageCliqueUnittest,
+      misc_unittest.GritNodeUnittest,
+      rc_unittest.RcUnittest,
+      tr_html_unittest.ParserUnittest,
+      tr_html_unittest.TrHtmlUnittest,
+      message_unittest.MessageUnittest,
+      tclib_unittest.TclibUnittest,
+      grit.format.rc_unittest.FormatRcUnittest,
+      rc2grd_unittest.Rc2GrdUnittest,
+      transl2tc_unittest.TranslationToTcUnittest,
+      txt_unittest.TxtUnittest,
+      admin_template_unittest.AdmGathererUnittest,
+      xtb_reader_unittest.XtbReaderUnittest,
+      misc_unittest.IfNodeUnittest,
+      util_unittest.UtilUnittest,
+      preprocess_unittest.PreProcessingUnittest,
+      postprocess_unittest.PostProcessingUnittest,
+      misc_unittest.ReleaseNodeUnittest,
+      shortcuts_unittests.ShortcutsUnittest,
+      muppet_strings_unittest.MuppetStringsUnittest,
+      filename_unittest.WindowsFilenameUnittest,
+      # add test classes here...
+    ]
+
+    for test_class in test_classes:
+      self.addTest(unittest.makeSuite(test_class))
+
+
+if __name__ == '__main__':
+  unittest.TextTestRunner(verbosity=2).run(TestSuiteAll())

tools/grit/grit/tool/__init__.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Package grit.tool
+'''
+
+pass

tools/grit/grit/tool/build.py

@@ -0,0 +1,232 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The 'grit build' tool along with integration for this tool with the
+SCons build system.
+'''
+
+import os
+import getopt
+import types
+import sys
+
+from grit import grd_reader
+from grit import util
+from grit.tool import interface
+from grit import shortcuts
+
+
+def ParseDefine(define):
+  '''Parses a define that is either like "NAME" or "NAME=VAL" and
+  returns its components, using True as the default value.  Values of
+  "1" and "0" are transformed to True and False respectively.
+  '''
+  parts = [part.strip() for part in define.split('=')]
+  assert len(parts) >= 1
+  name = parts[0]
+  val = True
+  if len(parts) > 1:
+    val = parts[1]
+  if val == "1": val = True
+  elif val == "0": val = False
+  return (name, val)
+
+
+class RcBuilder(interface.Tool):
+  '''A tool that builds RC files and resource header files for compilation.
+
+Usage:  grit build [-o OUTPUTDIR] [-D NAME[=VAL]]*
+
+All output options for this tool are specified in the input file (see
+'grit help' for details on how to specify the input file - it is a global
+option).
+
+Options:
+
+  -o OUTPUTDIR      Specify what directory output paths are relative to.
+                    Defaults to the current directory.
+  
+  -D NAME[=VAL]     Specify a C-preprocessor-like define NAME with optional
+                    value VAL (defaults to 1) which will be used to control
+                    conditional inclusion of resources.
+
+Conditional inclusion of resources only affects the output of files which
+control which resources get linked into a binary, e.g. it affects .rc files
+meant for compilation but it does not affect resource header files (that define
+IDs).  This helps ensure that values of IDs stay the same, that all messages
+are exported to translation interchange files (e.g. XMB files), etc.
+'''
+
+  def ShortDescription(self):
+    return 'A tool that builds RC files for compilation.'
+
+  def Run(self, opts, args):
+    self.output_directory = '.'
+    (own_opts, args) = getopt.getopt(args, 'o:D:')
+    for (key, val) in own_opts:
+      if key == '-o':
+        self.output_directory = val
+      elif key == '-D':
+        name, val = ParseDefine(val)
+        self.defines[name] = val
+    if len(args):
+      print "This tool takes no tool-specific arguments."
+      return 2
+    self.SetOptions(opts)
+    if self.scons_targets:
+      self.VerboseOut('Using SCons targets to identify files to output.\n')
+    else:
+      self.VerboseOut('Output directory: %s (absolute path: %s)\n' %
+                      (self.output_directory,
+                       os.path.abspath(self.output_directory)))
+    self.res = grd_reader.Parse(opts.input, debug=opts.extra_verbose)
+    self.res.RunGatherers(recursive = True)
+    self.Process()
+    return 0
+
+  def __init__(self):
+    # Default file-creation function is built-in file().  Only done to allow
+    # overriding by unit test.
+    self.fo_create = file
+    
+    # key/value pairs of C-preprocessor like defines that are used for
+    # conditional output of resources
+    self.defines = {}
+
+    # self.res is a fully-populated resource tree if Run()
+    # has been called, otherwise None.
+    self.res = None
+    
+    # Set to a list of filenames for the output nodes that are relative
+    # to the current working directory.  They are in the same order as the
+    # output nodes in the file.
+    self.scons_targets = None
+  
+  # static method
+  def ProcessNode(node, output_node, outfile):
+    '''Processes a node in-order, calling its formatter before and after
+    recursing to its children.
+    
+    Args:
+      node: grit.node.base.Node subclass
+      output_node: grit.node.io.File
+      outfile: open filehandle
+    '''
+    base_dir = util.dirname(output_node.GetOutputFilename())
+    
+    try:
+      formatter = node.ItemFormatter(output_node.GetType())
+      if formatter:
+        outfile.write(formatter.Format(node, output_node.GetLanguage(),
+                                       begin_item=True, output_dir=base_dir))
+    except:
+      print u"Error processing node %s" % unicode(node)
+      raise
+
+    for child in node.children:
+      RcBuilder.ProcessNode(child, output_node, outfile)
+
+    try:
+      if formatter:
+        outfile.write(formatter.Format(node, output_node.GetLanguage(),
+                                       begin_item=False, output_dir=base_dir))
+    except:
+      print u"Error processing node %s" % unicode(node)
+      raise
+  ProcessNode = staticmethod(ProcessNode)
+
+
+  def Process(self):
+    # Update filenames with those provided by SCons if we're being invoked
+    # from SCons.  The list of SCons targets also includes all <structure>
+    # node outputs, but it starts with our output files, in the order they
+    # occur in the .grd
+    if self.scons_targets:
+      assert len(self.scons_targets) >= len(self.res.GetOutputFiles())
+      outfiles = self.res.GetOutputFiles()
+      for ix in range(len(outfiles)):
+        outfiles[ix].output_filename = os.path.abspath(
+          self.scons_targets[ix])
+    else:
+      for output in self.res.GetOutputFiles():
+        output.output_filename = os.path.abspath(os.path.join(
+          self.output_directory, output.GetFilename()))
+    
+    for output in self.res.GetOutputFiles():
+      self.VerboseOut('Creating %s...' % output.GetFilename())
+      # Microsoft's RC compiler can only deal with single-byte or double-byte
+      # files (no UTF-8), so we make all RC files UTF-16 to support all
+      # character sets.
+      if output.GetType() in ['rc_header']:
+        encoding = 'cp1252'
+        outname = output.GetOutputFilename()
+        oldname = outname + '.tmp'
+        if os.access(oldname, os.F_OK):
+          os.remove(oldname)
+        try:
+          os.rename(outname, oldname)
+        except OSError:
+          oldname = None
+      else:
+        encoding = 'utf_16'
+      outfile = util.WrapOutputStream(
+        self.fo_create(output.GetOutputFilename(), 'wb'),
+        encoding)
+      
+      # Set the context, for conditional inclusion of resources
+      self.res.SetOutputContext(output.GetLanguage(), self.defines)
+      
+      # TODO(joi) Handle this more gracefully
+      import grit.format.rc_header
+      grit.format.rc_header.Item.ids_ = {}
+      
+      # Iterate in-order through entire resource tree, calling formatters on
+      # the entry into a node and on exit out of it.
+      self.ProcessNode(self.res, output, outfile)
+      
+      outfile.close()
+      if output.GetType() in ['rc_header'] and oldname:
+        if open(oldname).read() != open(outname).read():
+          os.remove(oldname)
+        else:
+          os.remove(outname)
+          os.rename(oldname, outname)
+      self.VerboseOut(' done.\n')
+    
+    # Print warnings if there are any duplicate shortcuts.
+    print '\n'.join(shortcuts.GenerateDuplicateShortcutsWarnings(
+      self.res.UberClique(), self.res.GetTcProject()))
+    
+    # Print out any fallback warnings, and missing translation errors, and
+    # exit with an error code if there are missing translations in a non-pseudo
+    # build
+    print self.res.UberClique().MissingTranslationsReport()
+    if self.res.UberClique().HasMissingTranslations():
+      sys.exit(-1)

tools/grit/grit/tool/count.py

@@ -0,0 +1,68 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Count number of occurrences of a given message ID
+'''
+
+import getopt
+import os
+import types
+
+from grit.tool import interface
+from grit import grd_reader
+from grit import util
+
+from grit.extern import tclib
+
+
+class CountMessage(interface.Tool):
+  '''Count the number of times a given message ID is used.
+'''
+
+  def __init__(self):
+    pass
+
+  def ShortDescription(self):
+    return 'Exports all translateable messages into an XMB file.'
+  
+  def Run(self, opts, args):
+    self.SetOptions(opts)
+    
+    id = args[0]
+    res_tree = grd_reader.Parse(opts.input, debug=opts.extra_verbose)
+    res_tree.OnlyTheseTranslations([])
+    res_tree.RunGatherers(True)
+    
+    count = 0
+    for c in res_tree.UberClique().AllCliques():
+      if c.GetId() == id:
+        count += 1
+    
+    print "There are %d occurrences of message %s." % (count, id)

tools/grit/grit/tool/diff_structures.py

@@ -0,0 +1,139 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The 'grit sdiff' tool.
+'''
+
+import os
+import getopt
+import tempfile
+
+from grit.node import structure
+from grit.tool import interface
+
+from grit import constants
+from grit import util
+
+# Builds the description for the tool (used as the __doc__
+# for the DiffStructures class).
+_class_doc = """\
+Allows you to view the differences in the structure of two files,
+disregarding their translateable content.  Translateable portions of
+each file are changed to the string "TTTTTT" before invoking the diff program
+specified by the P4DIFF environment variable.
+
+Usage: grit sdiff [-t TYPE] [-s SECTION] [-e ENCODING] LEFT RIGHT
+
+LEFT and RIGHT are the files you want to diff.  SECTION is required
+for structure types like 'dialog' to identify the part of the file to look at.
+ENCODING indicates the encoding of the left and right files (default 'cp1252').
+TYPE can be one of the following, defaults to 'tr_html':
+"""
+for gatherer in structure._GATHERERS:
+  _class_doc += " - %s\n" % gatherer
+
+
+class DiffStructures(interface.Tool):
+  __doc__ = _class_doc
+  
+  def __init__(self):
+    self.section = None
+    self.left_encoding = 'cp1252'
+    self.right_encoding = 'cp1252'
+    self.structure_type = 'tr_html'
+
+  def ShortDescription(self):
+    return 'View differences without regard for translateable portions.'
+
+  def Run(self, global_opts, args):
+    (opts, args) = getopt.getopt(args, 's:e:t:',
+                                 ['left_encoding=', 'right_encoding='])
+    for key, val in opts:
+      if key == '-s':
+        self.section = val
+      elif key == '-e':
+        self.left_encoding = val
+        self.right_encoding = val
+      elif key == '-t':
+        self.structure_type = val
+      elif key == '--left_encoding':
+        self.left_encoding = val
+      elif key == '--right_encoding':
+        self.right_encoding == val
+
+    if len(args) != 2:
+      print "Incorrect usage - 'grit help sdiff' for usage details."
+      return 2
+    
+    if 'P4DIFF' not in os.environ:
+      print "Environment variable P4DIFF not set; defaulting to 'windiff'."
+      diff_program = 'windiff'
+    else:
+      diff_program = os.environ['P4DIFF']
+    
+    left_trans = self.MakeStaticTranslation(args[0], self.left_encoding)
+    try:
+      try:
+        right_trans = self.MakeStaticTranslation(args[1], self.right_encoding)
+        
+        os.system('%s %s %s' % (diff_program, left_trans, right_trans))
+      finally:
+        os.unlink(right_trans)
+    finally:
+      os.unlink(left_trans)
+      
+  def MakeStaticTranslation(self, original_filename, encoding):
+    """Given the name of the structure type (self.structure_type), the filename
+    of the file holding the original structure, and optionally the "section" key
+    identifying the part of the file to look at (self.section), creates a
+    temporary file holding a "static" translation of the original structure
+    (i.e. one where all translateable parts have been replaced with "TTTTTT")
+    and returns the temporary file name.  It is the caller's responsibility to
+    delete the file when finished.
+    
+    Args:
+      original_filename: 'c:\\bingo\\bla.rc'
+    
+    Return:
+      'c:\\temp\\werlkjsdf334.tmp'
+    """
+    original = structure._GATHERERS[self.structure_type].FromFile(
+      original_filename, extkey=self.section, encoding=encoding)
+    original.Parse()
+    translated = original.Translate(constants.CONSTANT_LANGUAGE, False)
+    
+    fname = tempfile.mktemp()
+    fhandle = file(fname, 'w')
+    writer = util.WrapOutputStream(fhandle)
+    writer.write("Original filename: %s\n=============\n\n" % original_filename)
+    writer.write(translated)  # write in UTF-8
+    fhandle.close()
+    
+    return fname

tools/grit/grit/tool/interface.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Base class and interface for tools.
+'''
+
+import sys
+
+class Tool(object):
+  '''Base class for all tools.  Tools should use their docstring (i.e. the
+  class-level docstring) for the help they want to have printed when they
+  are invoked.'''
+
+  #
+  # Interface (abstract methods)
+  #
+
+  def ShortDescription(self):
+    '''Returns a short description of the functionality of the tool.'''
+    raise NotImplementedError()
+    
+  def Run(self, global_options, my_arguments):
+    '''Runs the tool.
+    
+    Args:
+      global_options: object grit_runner.Options
+      my_arguments: [arg1 arg2 ...]
+    
+    Return:
+      0 for success, non-0 for error
+    '''
+    raise NotImplementedError()
+
+  #
+  # Base class implementation
+  #
+  
+  def __init__(self):
+    self.o = None
+  
+  def SetOptions(self, opts):
+    self.o = opts
+  
+  def Out(self, text):
+    '''Always writes out 'text'.'''
+    self.o.output_stream.write(text)
+  
+  def VerboseOut(self, text):
+    '''Writes out 'text' if the verbose option is on.'''
+    if self.o.verbose:
+      self.o.output_stream.write(text)
+  
+  def ExtraVerboseOut(self, text):
+    '''Writes out 'text' if the extra-verbose option is on.
+    '''
+    if self.o.extra_verbose:
+      self.o.output_stream.write(text)

tools/grit/grit/tool/menu_from_parts.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The 'grit menufromparts' tool.'''
+
+import os
+import getopt
+import types
+
+from grit.tool import interface
+from grit.tool import transl2tc
+from grit import grd_reader
+from grit import tclib
+from grit import util
+from grit import xtb_reader
+
+
+import grit.extern.tclib
+
+
+class MenuTranslationsFromParts(interface.Tool):
+  '''One-off tool to generate translated menu messages (where each menu is kept
+in a single message) based on existing translations of the individual menu
+items.  Was needed when changing menus from being one message per menu item
+to being one message for the whole menu.'''
+
+  def ShortDescription(self):
+    return ('Create translations of whole menus from existing translations of '
+            'menu items.')
+  
+  def Run(self, globopt, args):
+    self.SetOptions(globopt)
+    assert len(args) == 2, "Need exactly two arguments, the XTB file and the output file"
+    
+    xtb_file = args[0]
+    output_file = args[1]
+    
+    grd = grd_reader.Parse(self.o.input, debug=self.o.extra_verbose)
+    grd.OnlyTheseTranslations([])  # don't load translations
+    grd.RunGatherers(recursive = True)
+    
+    xtb = {}
+    def Callback(msg_id, parts):
+      msg = []
+      for part in parts:
+        if part[0]:
+          msg = []
+          break  # it had a placeholder so ignore it
+        else:
+          msg.append(part[1])
+      if len(msg):
+        xtb[msg_id] = ''.join(msg)
+    f = file(xtb_file)
+    xtb_reader.Parse(f, Callback)
+    f.close()
+    
+    translations = []  # list of translations as per transl2tc.WriteTranslations
+    for node in grd:
+      if node.name == 'structure' and node.attrs['type'] == 'menu':
+        assert len(node.GetCliques()) == 1
+        message = node.GetCliques()[0].GetMessage()
+        translation = []
+        
+        contents = message.GetContent()
+        for part in contents:
+          if isinstance(part, types.StringTypes):
+            id = grit.extern.tclib.GenerateMessageId(part)
+            if id not in xtb:
+              print "WARNING didn't find all translations for menu %s" % node.attrs['name']
+              translation = []
+              break
+            translation.append(xtb[id])
+          else:
+            translation.append(part.GetPresentation())
+        
+        if len(translation):
+          translations.append([message.GetId(), ''.join(translation)])
+    
+    f = util.WrapOutputStream(file(output_file, 'w'))
+    transl2tc.TranslationToTc.WriteTranslations(f, translations)
+    f.close()

tools/grit/grit/tool/newgrd.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Tool to create a new, empty .grd file with all the basic sections.
+'''
+
+from grit.tool import interface
+from grit import constants
+from grit import util
+
+# The contents of the new .grd file
+_FILE_CONTENTS = '''\
+<?xml version="1.0" encoding="UTF-8"?>
+<grit base_dir="." latest_public_release="0" current_release="1"
+      source_lang_id="en" enc_check="%s">
+  <outputs>
+    <!-- TODO add each of your output files.  Modify the three below, and add
+    your own for your various languages.  See the user's guide for more
+    details.
+    Note that all output references are relative to the output directory
+    which is specified at build time. -->
+    <output filename="resource.h" type="rc_header" />
+    <output filename="en_resource.rc" type="rc_all" />
+    <output filename="fr_resource.rc" type="rc_all" />
+  </outputs>
+  <translations>
+    <!-- TODO add references to each of the XTB files (from the Translation
+    Console) that contain translations of messages in your project.  Each
+    takes a form like <file path="english.xtb" />.  Remember that all file
+    references are relative to this .grd file. -->
+  </translations>
+  <release seq="1">
+    <includes>
+      <!-- TODO add a list of your included resources here, e.g. BMP and GIF
+      resources. -->
+    </includes>
+    <structures>
+      <!-- TODO add a list of all your structured resources here, e.g. HTML
+      templates, menus, dialogs etc.  Note that for menus, dialogs and version
+      information resources you reference an .rc file containing them.-->
+    </structures>
+    <messages>
+      <!-- TODO add all of your "string table" messages here.  Remember to
+      change nontranslateable parts of the messages into placeholders (using the
+      <ph> element).  You can also use the 'grit add' tool to help you identify
+      nontranslateable parts and create placeholders for them. -->
+    </messages>
+  </release>
+</grit>''' % constants.ENCODING_CHECK
+
+
+class NewGrd(interface.Tool):
+  '''Usage: grit newgrd OUTPUT_FILE
+
+Creates a new, empty .grd file OUTPUT_FILE with comments about what to put
+where in the file.'''
+
+  def ShortDescription(self):
+    return 'Create a new empty .grd file.'
+  
+  def Run(self, global_options, my_arguments):
+    if not len(my_arguments) == 1:
+      print 'This tool requires exactly one argument, the name of the output file.'
+      return 2
+    filename = my_arguments[0]
+    out = util.WrapOutputStream(file(filename, 'w'), 'utf-8')
+    out.write(_FILE_CONTENTS)
+    out.close()
+    print "Wrote file %s" % filename

tools/grit/grit/tool/postprocess_interface.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+''' Base class for postprocessing of RC files.
+'''
+
+import sys
+
+class PostProcessor(object):
+  ''' Base class for postprocessing of the RC file data before being
+  output through the RC2GRD tool. You should implement this class if
+  you want GRIT to do specific things to the RC files after it has
+  converted the data into GRD format, i.e. change the content of the
+  RC file, and put it into a P4 changelist, etc.'''
+
+
+  def Process(self, rctext, rcpath, grdnode):
+    ''' Processes the data in rctext and grdnode.
+    Args:
+      rctext: string containing the contents of the RC file being processed.
+      rcpath: the path used to access the file.
+      grdtext: the root node of the grd xml data generated by 
+      the rc2grd tool.
+    
+    Return:
+      The root node of the processed GRD tree.
+    '''
+    raise NotImplementedError()
+
+

tools/grit/grit/tool/postprocess_unittest.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit test that checks postprocessing of files.
+   Tests postprocessing by having the postprocessor
+   modify the grd data tree, changing the message name attributes.
+'''
+
+import os
+import re
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+
+import grit.tool.postprocess_interface
+from grit.tool import rc2grd
+
+
+class PostProcessingUnittest(unittest.TestCase):
+  
+  def testPostProcessing(self):
+    rctext = '''STRINGTABLE
+BEGIN
+  DUMMY_STRING_1         "String 1"
+  // Some random description 
+  DUMMY_STRING_2        "This text was added during preprocessing"
+END  
+    '''
+    tool = rc2grd.Rc2Grd()
+    class DummyOpts(object):
+      verbose = False
+      extra_verbose = False
+    tool.o = DummyOpts()  
+    tool.post_process = 'grit.tool.postprocess_unittest.DummyPostProcessor'
+    result = tool.Process(rctext, '.\resource.rc')
+    
+    self.failUnless(
+      result.children[2].children[2].children[0].attrs['name'] == 'SMART_STRING_1')
+    self.failUnless(
+      result.children[2].children[2].children[1].attrs['name'] == 'SMART_STRING_2')
+    
+class DummyPostProcessor(grit.tool.postprocess_interface.PostProcessor):
+  '''
+  Post processing replaces all message name attributes containing "DUMMY" to
+  "SMART".
+  '''
+  def Process(self, rctext, rcpath, grdnode):
+    smarter = re.compile(r'(DUMMY)(.*)')
+    messages = grdnode.children[2].children[2]
+    for node in messages.children:
+      name_attr = node.attrs['name']
+      m = smarter.search(name_attr)
+      if m:
+         node.attrs['name'] = 'SMART' + m.group(2)
+    return grdnode 
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/tool/preprocess_interface.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+''' Base class for preprocessing of RC files.
+'''
+
+import sys
+
+class PreProcessor(object):
+  ''' Base class for preprocessing of the RC file data before being
+  output through the RC2GRD tool. You should implement this class if
+  you have specific constructs in your RC files that GRIT cannot handle.'''
+
+
+  def Process(self, rctext, rcpath):
+    ''' Processes the data in rctext.
+    Args:
+      rctext: string containing the contents of the RC file being processed
+      rcpath: the path used to access the file.
+    
+    Return:
+      The processed text.
+    '''
+    raise NotImplementedError()
+
+

tools/grit/grit/tool/preprocess_unittest.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit test that checks preprocessing of files.
+   Tests preprocessing by adding having the preprocessor
+   provide the actual rctext data.
+'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+
+import grit.tool.preprocess_interface
+from grit.tool import rc2grd
+
+
+class PreProcessingUnittest(unittest.TestCase):
+  
+  def testPreProcessing(self):
+    tool = rc2grd.Rc2Grd()
+    class DummyOpts(object):
+      verbose = False
+      extra_verbose = False
+    tool.o = DummyOpts()  
+    tool.pre_process = 'grit.tool.preprocess_unittest.DummyPreProcessor'
+    result = tool.Process('', '.\resource.rc')
+    
+    self.failUnless(
+      result.children[2].children[2].children[0].attrs['name'] == 'DUMMY_STRING_1')
+    
+class DummyPreProcessor(grit.tool.preprocess_interface.PreProcessor):
+  def Process(self, rctext, rcpath):
+    rctext = '''STRINGTABLE
+BEGIN
+  DUMMY_STRING_1         "String 1"
+  // Some random description 
+  DUMMY_STRING_2        "This text was added during preprocessing"
+END  
+    '''
+    return rctext
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/tool/rc2grd.py

@@ -0,0 +1,427 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The 'grit rc2grd' tool.'''
+
+
+import os.path
+import getopt
+import re
+import StringIO
+import types
+
+import grit.node.empty
+from grit.node import include
+from grit.node import structure
+from grit.node import message
+
+from grit.gather import rc
+from grit.gather import tr_html
+
+from grit.tool import interface
+from grit.tool import postprocess_interface
+from grit.tool import preprocess_interface
+
+from grit import grd_reader
+from grit import tclib
+from grit import util
+
+
+# Matches files referenced from an .rc file
+_FILE_REF = re.compile('''
+  ^(?P<id>[A-Z_0-9.]+)[ \t]+
+  (?P<type>[A-Z_0-9]+)[ \t]+
+  "(?P<file>.*?([^"]|""))"[ \t]*$''', re.VERBOSE | re.MULTILINE)
+
+
+# Matches a dialog section
+_DIALOG = re.compile('^(?P<id>[A-Z0-9_]+)\s+DIALOG(EX)?\s.+?^BEGIN\s*$.+?^END\s*$',
+                     re.MULTILINE | re.DOTALL)
+
+
+# Matches a menu section
+_MENU = re.compile('^(?P<id>[A-Z0-9_]+)\s+MENU.+?^BEGIN\s*$.+?^END\s*$',
+                   re.MULTILINE | re.DOTALL)
+
+
+# Matches a versioninfo section
+_VERSIONINFO = re.compile('^(?P<id>[A-Z0-9_]+)\s+VERSIONINFO\s.+?^BEGIN\s*$.+?^END\s*$',
+                          re.MULTILINE | re.DOTALL)
+
+
+# Matches a stringtable
+_STRING_TABLE = re.compile('^STRINGTABLE(\s+(PRELOAD|DISCARDABLE|CHARACTERISTICS.+|LANGUAGE.+|VERSION.+))*\s*\nBEGIN\s*$(?P<body>.+?)^END\s*$',
+                           re.MULTILINE | re.DOTALL)
+
+
+# Matches each message inside a stringtable, breaking it up into comments,
+# the ID of the message, and the (RC-escaped) message text.
+_MESSAGE = re.compile('''
+  (?P<comment>(^\s+//.+?)*)  # 0 or more lines of comments preceding the message
+  ^\s*
+  (?P<id>[A-Za-z0-9_]+)  # id
+  \s+
+  "(?P<text>.*?([^"]|""))"([^"]|$)  # The message itself
+  ''', re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+# Matches each line of comment text in a multi-line comment.
+_COMMENT_TEXT = re.compile('^\s*//\s*(?P<text>.+?)$', re.MULTILINE)
+
+
+# Matches a string that is empty or all whitespace
+_WHITESPACE_ONLY = re.compile('\A\s*\Z', re.MULTILINE)
+
+
+# Finds printf and FormatMessage style format specifiers
+# Uses non-capturing groups except for the outermost group, so the output of
+# re.split() should include both the normal text and what we intend to
+# replace with placeholders.
+# TODO(joi) Check documentation for printf (and Windows variants) and FormatMessage
+_FORMAT_SPECIFIER = re.compile(
+  '(%[-# +]?(?:[0-9]*|\*)(?:\.(?:[0-9]+|\*))?(?:h|l|L)?' # printf up to last char
+  '(?:d|i|o|u|x|X|e|E|f|F|g|G|c|r|s|ls|ws)'              # printf last char
+  '|\$[1-9][0-9]*)')                                     # FormatMessage
+
+
+class Rc2Grd(interface.Tool):
+  '''A tool for converting .rc files to .grd files.  This tool is only for
+converting the source (nontranslated) .rc file to a .grd file.  For importing
+existing translations, use the rc2xtb tool.
+
+Usage:  grit [global options] rc2grd [OPTIONS] RCFILE
+
+The tool takes a single argument, which is the path to the .rc file to convert.
+It outputs a .grd file with the same name in the same directory as the .rc file.
+The .grd file may have one or more TODO comments for things that have to be
+cleaned up manually.
+
+OPTIONS may be any of the following:
+
+  -e ENCODING    Specify the ENCODING of the .rc file. Default is 'cp1252'.
+  
+  -h TYPE        Specify the TYPE attribute for HTML structures.
+                 Default is 'tr_html'.
+
+  -u ENCODING    Specify the ENCODING of HTML files. Default is 'utf-8'.
+
+  -n MATCH       Specify the regular expression to match in comments that will
+                 indicate that the resource the comment belongs to is not
+                 translateable. Default is 'Not locali(s|z)able'.
+
+  -r GRDFILE     Specify that GRDFILE should be used as a "role model" for
+                 any placeholders that otherwise would have had TODO names.
+                 This attempts to find an identical message in the GRDFILE
+                 and uses that instead of the automatically placeholderized
+                 message.
+
+  --pre CLASS    Specify an optional, fully qualified classname, which
+                 has to be a subclass of grit.tool.PreProcessor, to
+                 run on the text of the RC file before conversion occurs.
+                 This can be used to support constructs in the RC files
+                 that GRIT cannot handle on its own.
+
+  --post CLASS   Specify an optional, fully qualified classname, which
+                 has to be a subclass of grit.tool.PostProcessor, to
+                 run on the text of the converted RC file.
+                 This can be used to alter the content of the RC file
+                 based on the conversion that occured.
+
+For menus, dialogs and version info, the .grd file will refer to the original
+.rc file.  Once conversion is complete, you can strip the original .rc file
+of its string table and all comments as these will be available in the .grd
+file.
+
+Note that this tool WILL NOT obey C preprocessor rules, so even if something
+is #if 0-ed out it will still be included in the output of this tool
+Therefore, if your .rc file contains sections like this, you should run the
+C preprocessor on the .rc file or manually edit it before using this tool.
+'''
+  
+  def ShortDescription(self):
+    return 'A tool for converting .rc source files to .grd files.'
+  
+  def __init__(self):
+    self.input_encoding = 'cp1252'
+    self.html_type = 'tr_html'
+    self.html_encoding = 'utf-8'
+    self.not_localizable_re = re.compile('Not locali(s|z)able')
+    self.role_model = None
+    self.pre_process = None
+    self.post_process = None
+  
+  def ParseOptions(self, args):
+    '''Given a list of arguments, set this object's options and return
+    all non-option arguments.
+    '''
+    (own_opts, args) = getopt.getopt(args, 'e:h:u:n:r', ['pre=', 'post='])
+    for (key, val) in own_opts:
+      if key == '-e':
+        self.input_encoding = val
+      elif key == '-h':
+        self.html_type = val
+      elif key == '-u':
+        self.html_encoding = val
+      elif key == '-n':
+        self.not_localizable_re = re.compile(val)
+      elif key == '-r':
+        self.role_model = grd_reader.Parse(val)
+      elif key == '--pre':
+        self.pre_process = val
+      elif key == '--post':
+        self.post_process = val
+    return args
+
+  def Run(self, opts, args):
+    args = self.ParseOptions(args)
+    if len(args) != 1:
+      print ('This tool takes a single tool-specific argument, the path to the\n'
+             '.rc file to process.')
+      return 2
+    self.SetOptions(opts)
+    
+    path = args[0]
+    out_path = os.path.join(util.dirname(path),
+                os.path.splitext(os.path.basename(path))[0] + '.grd')
+    
+    rcfile = util.WrapInputStream(file(path, 'r'), self.input_encoding)
+    rctext = rcfile.read()
+    
+    grd_text = unicode(self.Process(rctext, path))
+
+    rcfile.close()
+    
+    outfile = util.WrapOutputStream(file(out_path, 'w'), 'utf-8')
+    outfile.write(grd_text)
+    outfile.close()
+    
+    print 'Wrote output file %s.\nPlease check for TODO items in the file.' % out_path
+
+  
+  def Process(self, rctext, rc_path):
+    '''Processes 'rctext' and returns a resource tree corresponding to it.
+    
+    Args:
+      rctext: complete text of the rc file
+      rc_path: 'resource\resource.rc'
+    
+    Return:
+      grit.node.base.Node subclass
+    '''
+
+    if self.pre_process:
+      preprocess_class = util.NewClassInstance(self.pre_process,
+                                               preprocess_interface.PreProcessor)
+      if preprocess_class:
+        rctext = preprocess_class.Process(rctext, rc_path)
+      else:
+        self.Out(
+          'PreProcessing class could not be found. Skipping preprocessing.\n')
+
+    # Start with a basic skeleton for the .grd file
+    root = grd_reader.Parse(StringIO.StringIO(
+      '''<?xml version="1.0" encoding="UTF-8"?>
+      <grit base_dir="." latest_public_release="0"
+          current_release="1" source_lang_id="en">
+        <outputs />
+        <translations />
+        <release seq="1">
+          <includes />
+          <structures />
+          <messages />
+        </release>
+      </grit>'''), util.dirname(rc_path))
+    includes = root.children[2].children[0]
+    structures = root.children[2].children[1]
+    messages = root.children[2].children[2]
+    assert (isinstance(includes, grit.node.empty.IncludesNode) and
+            isinstance(structures, grit.node.empty.StructuresNode) and
+            isinstance(messages, grit.node.empty.MessagesNode))
+    
+    self.AddIncludes(rctext, includes)
+    self.AddStructures(rctext, structures, os.path.basename(rc_path))
+    self.AddMessages(rctext, messages)
+    
+    self.VerboseOut('Validating that all IDs are unique...\n')
+    root.ValidateUniqueIds()
+    self.ExtraVerboseOut('Done validating that all IDs are unique.\n')
+    
+    if self.post_process:
+      postprocess_class = util.NewClassInstance(self.post_process,
+                                                postprocess_interface.PostProcessor)
+      if postprocess_class:
+        root = postprocess_class.Process(rctext, rc_path, root)
+      else:
+        self.Out(
+          'PostProcessing class could not be found. Skipping postprocessing.\n')
+
+    return root
+
+
+  def AddIncludes(self, rctext, node):
+    '''Scans 'rctext' for included resources (e.g. BITMAP, ICON) and
+    adds each included resource as an <include> child node of 'node'.'''
+    for m in _FILE_REF.finditer(rctext):
+      id = m.group('id')
+      type = m.group('type').upper()
+      fname = rc.Section.UnEscape(m.group('file'))
+      assert fname.find('\n') == -1
+      if type != 'HTML':
+        self.VerboseOut('Processing %s with ID %s (filename: %s)\n' % (type, id, fname))
+        node.AddChild(include.IncludeNode.Construct(node, id, type, fname))
+  
+  
+  def AddStructures(self, rctext, node, rc_filename):
+    '''Scans 'rctext' for structured resources (e.g. menus, dialogs, version
+    information resources and HTML templates) and adds each as a <structure>
+    child of 'node'.'''
+    # First add HTML includes
+    for m in _FILE_REF.finditer(rctext):
+      id = m.group('id')
+      type = m.group('type').upper()
+      fname = rc.Section.UnEscape(m.group('file'))
+      if type == 'HTML':
+        node.AddChild(structure.StructureNode.Construct(
+          node, id, self.html_type, fname, self.html_encoding))
+    
+    # Then add all RC includes
+    def AddStructure(type, id):
+      self.VerboseOut('Processing %s with ID %s\n' % (type, id))
+      node.AddChild(structure.StructureNode.Construct(node, id, type,
+                                                      rc_filename,
+                                                      encoding=self.input_encoding))
+    for m in _MENU.finditer(rctext):
+      AddStructure('menu', m.group('id'))
+    for m in _DIALOG.finditer(rctext):
+      AddStructure('dialog', m.group('id'))
+    for m in _VERSIONINFO.finditer(rctext):
+      AddStructure('version', m.group('id'))
+  
+  
+  def AddMessages(self, rctext, node):
+    '''Scans 'rctext' for all messages in string tables, preprocesses them as
+    much as possible for placeholders (e.g. messages containing $1, $2 or %s, %d
+    type format specifiers get those specifiers replaced with placeholders, and
+    HTML-formatted messages get run through the HTML-placeholderizer).  Adds
+    each message as a <message> node child of 'node'.'''
+    for tm in _STRING_TABLE.finditer(rctext):
+      table = tm.group('body')
+      for mm in _MESSAGE.finditer(table):
+        comment_block = mm.group('comment')
+        comment_text = []
+        for cm in _COMMENT_TEXT.finditer(comment_block):
+          comment_text.append(cm.group('text'))
+        comment_text = ' '.join(comment_text)
+        
+        id = mm.group('id')
+        text = rc.Section.UnEscape(mm.group('text'))
+        
+        self.VerboseOut('Processing message %s (text: "%s")\n' % (id, text))
+        
+        msg_obj = self.Placeholderize(text)
+        
+        # Messages that contain only placeholders do not need translation.
+        is_translateable = False
+        for item in msg_obj.GetContent():
+          if isinstance(item, types.StringTypes):
+            if not _WHITESPACE_ONLY.match(item):
+              is_translateable = True
+        
+        if self.not_localizable_re.search(comment_text):
+          is_translateable = False
+        
+        message_meaning = ''
+        internal_comment = ''
+        
+        # If we have a "role model" (existing GRD file) and this node exists
+        # in the role model, use the description, meaning and translateable
+        # attributes from the role model.
+        if self.role_model:
+          role_node = self.role_model.GetNodeById(id)
+          if role_node:
+            is_translateable = role_node.IsTranslateable()
+            message_meaning = role_node.attrs['meaning']
+            comment_text = role_node.attrs['desc']
+            internal_comment = role_node.attrs['internal_comment']
+        
+        # For nontranslateable messages, we don't want the complexity of
+        # placeholderizing everything.
+        if not is_translateable:
+          msg_obj = tclib.Message(text=text)
+        
+        msg_node = message.MessageNode.Construct(node, msg_obj, id,
+                                                 desc=comment_text,
+                                                 translateable=is_translateable,
+                                                 meaning=message_meaning)
+        msg_node.attrs['internal_comment'] = internal_comment
+        
+        node.AddChild(msg_node)
+        self.ExtraVerboseOut('Done processing message %s\n' % id)
+  
+  
+  def Placeholderize(self, text):
+    '''Creates a tclib.Message object from 'text', attempting to recognize
+    a few different formats of text that can be automatically placeholderized
+    (HTML code, printf-style format strings, and FormatMessage-style format
+    strings).
+    '''
+    
+    try:
+      # First try HTML placeholderizing.
+      # TODO(joi) Allow use of non-TotalRecall flavors of HTML placeholderizing
+      msg = tr_html.HtmlToMessage(text, True)
+      for item in msg.GetContent():
+        if not isinstance(item, types.StringTypes):
+          return msg  # Contained at least one placeholder, so we're done
+          
+      # HTML placeholderization didn't do anything, so try to find printf or
+      # FormatMessage format specifiers and change them into placeholders.
+      msg = tclib.Message()
+      parts = _FORMAT_SPECIFIER.split(text)
+      todo_counter = 1  # We make placeholder IDs 'TODO_0001' etc.
+      for part in parts:
+        if _FORMAT_SPECIFIER.match(part):
+          msg.AppendPlaceholder(tclib.Placeholder(
+            'TODO_%04d' % todo_counter, part, 'TODO'))
+          todo_counter += 1
+        elif part != '':
+          msg.AppendText(part)
+      
+      if self.role_model and len(parts) > 1:  # there are TODO placeholders
+        role_model_msg = self.role_model.UberClique().BestCliqueByOriginalText(
+          msg.GetRealContent(), '')
+        if role_model_msg:
+          # replace wholesale to get placeholder names and examples
+          msg = role_model_msg
+      
+      return msg
+    except:
+      print 'Exception processing message with text "%s"' % text
+      raise

tools/grit/grit/tool/rc2grd_unittest.py

@@ -0,0 +1,162 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.tool.rc2grd'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import re
+import StringIO
+import unittest
+
+from grit.node import base
+from grit.tool import rc2grd
+from grit.gather import rc
+from grit import grd_reader
+
+
+class Rc2GrdUnittest(unittest.TestCase):
+  def testPlaceholderize(self):
+    tool = rc2grd.Rc2Grd()
+    original = "Hello %s, how are you? I'm $1 years old!"
+    msg = tool.Placeholderize(original)
+    self.failUnless(msg.GetPresentableContent() == "Hello TODO_0001, how are you? I'm TODO_0002 years old!")
+    self.failUnless(msg.GetRealContent() == original)
+  
+  def testHtmlPlaceholderize(self):
+    tool = rc2grd.Rc2Grd()
+    original = "Hello <b>[USERNAME]</b>, how are you? I'm [AGE] years old!"
+    msg = tool.Placeholderize(original)
+    self.failUnless(msg.GetPresentableContent() ==
+                    "Hello BEGIN_BOLDX_USERNAME_XEND_BOLD, how are you? I'm X_AGE_X years old!")
+    self.failUnless(msg.GetRealContent() == original)
+  
+  def testMenuWithoutWhitespaceRegression(self):
+    # There was a problem in the original regular expression for parsing out
+    # menu sections, that would parse the following block of text as a single
+    # menu instead of two.
+    two_menus = '''
+// Hyper context menus
+IDR_HYPERMENU_FOLDER MENU
+BEGIN
+    POPUP "HyperFolder"
+    BEGIN
+        MENUITEM "Open Containing Folder",      IDM_OPENFOLDER
+    END
+END
+
+IDR_HYPERMENU_FILE MENU
+BEGIN
+    POPUP "HyperFile"
+    BEGIN
+        MENUITEM "Open Folder",                 IDM_OPENFOLDER
+    END
+END
+
+'''
+    self.failUnless(len(rc2grd._MENU.findall(two_menus)) == 2)
+
+  def testRegressionScriptWithTranslateable(self):
+    tool = rc2grd.Rc2Grd()
+
+    # test rig
+    class DummyNode(base.Node):
+      def AddChild(self, item):
+        self.node = item
+      verbose = False
+      extra_verbose = False
+    tool.not_localizable_re = re.compile('')
+    tool.o = DummyNode()
+    
+    rc_text = '''STRINGTABLE\nBEGIN\nID_BINGO "<SPAN id=hp style='BEHAVIOR: url(#default#homepage)'></SPAN><script>if (!hp.isHomePage('[$~HOMEPAGE~$]')) {document.write(""<a href=\\""[$~SETHOMEPAGEURL~$]\\"" >Set As Homepage</a> - "");}</script>"\nEND\n'''
+    tool.AddMessages(rc_text, tool.o)
+    self.failUnless(tool.o.node.GetCdata().find('Set As Homepage') != -1)
+    
+    # TODO(joi) Improve the HTML parser to support translateables inside
+    # <script> blocks?
+    self.failUnless(tool.o.node.attrs['translateable'] == 'false')
+
+  def testRoleModel(self):
+    rc_text = ('STRINGTABLE\n'
+               'BEGIN\n'
+               '  // This should not show up\n'
+               '  IDS_BINGO "Hello %s, how are you?"\n'
+               '  // The first description\n'
+               '  IDS_BONGO "Hello %s, my name is %s, and yours?"\n'
+               '  IDS_PROGRAMS_SHUTDOWN_TEXT      "Google Desktop Search needs to close the following programs:\\n\\n$1\\nThe installation will not proceed if you choose to cancel."\n'
+               'END\n')
+    tool = rc2grd.Rc2Grd()
+    tool.role_model = grd_reader.Parse(StringIO.StringIO(
+      '''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <release seq="3">
+          <messages>
+            <message name="IDS_BINGO">
+              Hello <ph name="USERNAME">%s<ex>Joi</ex></ph>, how are you?
+            </message>
+            <message name="IDS_BONGO" desc="The other description">
+              Hello <ph name="USERNAME">%s<ex>Jakob</ex></ph>, my name is <ph name="ADMINNAME">%s<ex>Joi</ex></ph>, and yours?
+            </message>
+            <message name="IDS_PROGRAMS_SHUTDOWN_TEXT" desc="LIST_OF_PROGRAMS is replaced by a bulleted list of program names.">
+              Google Desktop Search needs to close the following programs:
+
+<ph name="LIST_OF_PROGRAMS">$1<ex>Program 1, Program 2</ex></ph>
+The installation will not proceed if you choose to cancel.
+            </message>
+          </messages>
+        </release>
+      </grit>'''), dir='.')
+    
+    # test rig
+    class DummyOpts(object):
+      verbose = False
+      extra_verbose = False
+    tool.o = DummyOpts()
+    result = tool.Process(rc_text, '.\resource.rc')
+    self.failUnless(
+      result.children[2].children[2].children[0].attrs['desc'] == '')
+    self.failUnless(
+      result.children[2].children[2].children[0].children[0].attrs['name'] == 'USERNAME')
+    self.failUnless(
+      result.children[2].children[2].children[1].attrs['desc'] == 'The other description')
+    self.failUnless(
+      result.children[2].children[2].children[1].attrs['meaning'] == '')
+    self.failUnless(
+      result.children[2].children[2].children[1].children[0].attrs['name'] == 'USERNAME')
+    self.failUnless(
+      result.children[2].children[2].children[1].children[1].attrs['name'] == 'ADMINNAME')
+    self.failUnless(
+      result.children[2].children[2].children[2].children[0].attrs['name'] == 'LIST_OF_PROGRAMS')
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/tool/resize.py

@@ -0,0 +1,325 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The 'grit resize' tool.
+'''
+
+import getopt
+import os
+import types
+
+from grit.tool import interface
+from grit.tool import build
+from grit import grd_reader
+from grit import pseudo
+from grit import util
+
+from grit.node import include
+from grit.node import structure
+from grit.node import message
+
+from grit.format import rc_header
+
+
+# Template for the .vcproj file, with a couple of [[REPLACEABLE]] parts.
+PROJECT_TEMPLATE = '''\
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="7.10"
+	Name="[[DIALOG_NAME]]"
+	ProjectGUID="[[PROJECT_GUID]]"
+	Keyword="Win32Proj">
+	<Platforms>
+		<Platform
+			Name="Win32"/>
+	</Platforms>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="Debug"
+			IntermediateDirectory="Debug"
+			ConfigurationType="1"
+			CharacterSet="2">
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}">
+			<File
+				RelativePath=".\[[DIALOG_NAME]].rc">
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>'''
+
+
+# Template for the .rc file with a couple of [[REPLACEABLE]] parts.
+# TODO(joi) Improve this (and the resource.h template) to allow saving and then
+# reopening of the RC file in Visual Studio.  Currently you can only open it
+# once and change it, then after you close it you won't be able to reopen it.
+RC_TEMPLATE = '''\
+// Copyright (c) Google Inc. 2005
+// All rights reserved.
+// This file is automatically generated by GRIT and intended for editing
+// the layout of the dialogs contained in it.  Do not edit anything but the
+// dialogs.  Any changes made to translateable portions of the dialogs will
+// be ignored by GRIT.
+
+#include "resource.h"
+#include <winres.h>
+#include <winresrc.h>
+
+LANGUAGE LANG_NEUTRAL, SUBLANG_NEUTRAL
+
+#pragma code_page([[CODEPAGE_NUM]])
+
+[[INCLUDES]]
+
+[[DIALOGS]]
+'''
+
+
+# Template for the resource.h file with a couple of [[REPLACEABLE]] parts.
+HEADER_TEMPLATE = '''\
+// Copyright (c) Google Inc. 2005
+// All rights reserved.
+// This file is automatically generated by GRIT.  Do not edit.
+
+#pragma once
+
+// Edit commands
+#define ID_EDIT_CLEAR                   0xE120
+#define ID_EDIT_CLEAR_ALL               0xE121
+#define ID_EDIT_COPY                    0xE122
+#define ID_EDIT_CUT                     0xE123
+#define ID_EDIT_FIND                    0xE124
+#define ID_EDIT_PASTE                   0xE125
+#define ID_EDIT_PASTE_LINK              0xE126
+#define ID_EDIT_PASTE_SPECIAL           0xE127
+#define ID_EDIT_REPEAT                  0xE128
+#define ID_EDIT_REPLACE                 0xE129
+#define ID_EDIT_SELECT_ALL              0xE12A
+#define ID_EDIT_UNDO                    0xE12B
+#define ID_EDIT_REDO                    0xE12C
+
+
+[[DEFINES]]
+'''
+
+
+class ResizeDialog(interface.Tool):
+  '''Generates an RC file, header and Visual Studio project that you can use
+with Visual Studio's GUI resource editor to modify the layout of dialogs for
+the language of your choice.  You then use the RC file, after you resize the
+dialog, for the language or languages of your choice, using the <skeleton> child
+of the <structure> node for the dialog.  The translateable bits of the dialog
+will be ignored when you use the <skeleton> node (GRIT will instead use the
+translateable bits from the original dialog) but the layout changes you make
+will be used.  Note that your layout changes must preserve the order of the
+translateable elements in the RC file.
+
+Usage: grit resize [-f BASEFOLDER] [-l LANG] [-e RCENCODING] DIALOGID*
+
+Arguments:
+  DIALOGID        The 'name' attribute of a dialog to output for resizing.  Zero
+                  or more of these parameters can be used.  If none are
+                  specified, all dialogs from the input .grd file are output.
+
+Options:
+
+  -f BASEFOLDER   The project will be created in a subfolder of BASEFOLDER.
+                  The name of the subfolder will be the first DIALOGID you
+                  specify.  Defaults to '.'
+
+  -l LANG         Specifies that the RC file should contain a dialog translated
+                  into the language LANG.  The default is a cp1252-representable
+                  pseudotranslation, because Visual Studio's GUI RC editor only
+                  supports single-byte encodings.
+
+  -c CODEPAGE     Code page number to indicate to the RC compiler the encoding
+                  of the RC file, default is something reasonable for the
+                  language you selected (but this does not work for every single
+                  language).  See details on codepages below.  NOTE that you do
+                  not need to specify the codepage unless the tool complains
+                  that it's not sure which codepage to use.  See the following
+                  page for codepage numbers supported by Windows:
+                  http://www.microsoft.com/globaldev/reference/wincp.mspx
+
+  -D NAME[=VAL]   Specify a C-preprocessor-like define NAME with optional
+                  value VAL (defaults to 1) which will be used to control
+                  conditional inclusion of resources.
+
+
+IMPORTANT NOTE:  For now, the tool outputs a UTF-8 encoded file for any language
+that can not be represented in cp1252 (i.e. anything other than Western
+European languages).  You will need to open this file in a text editor and
+save it using the codepage indicated in the #pragma code_page(XXXX) command
+near the top of the file, before you open it in Visual Studio.
+
+'''
+
+  # TODO(joi) It would be cool to have this tool note the Perforce revision
+  # of the original RC file somewhere, such that the <skeleton> node could warn
+  # if the original RC file gets updated without the skeleton file being updated.
+  
+  # TODO(joi) Would be cool to have option to add the files to Perforce
+
+  def __init__(self):
+    self.lang = pseudo.PSEUDO_LANG
+    self.defines = {}
+    self.base_folder = '.'
+    self.codepage_number = 1252
+    self.codepage_number_specified_explicitly = False
+  
+  def SetLanguage(self, lang):
+    '''Sets the language code to output things in.
+    '''
+    self.lang = lang
+    if not self.codepage_number_specified_explicitly:
+      self.codepage_number = util.LanguageToCodepage(lang)
+
+  def GetEncoding(self):
+    if self.codepage_number == 1200:
+      return 'utf_16'
+    if self.codepage_number == 65001:
+      return 'utf_8'
+    return 'cp%d' % self.codepage_number
+
+  def ShortDescription(self):
+    return 'Generate a file where you can resize a given dialog.'
+  
+  def Run(self, opts, args):
+    self.SetOptions(opts)
+    
+    own_opts, args = getopt.getopt(args, 'l:f:c:D:')
+    for key, val in own_opts:
+      if key == '-l':
+        self.SetLanguage(val)
+      if key == '-f':
+        self.base_folder = val
+      if key == '-c':
+        self.codepage_number = int(val)
+        self.codepage_number_specified_explicitly = True
+      if key == '-D':
+        name, val = build.ParseDefine(val)
+        self.defines[name] = val
+    
+    res_tree = grd_reader.Parse(opts.input, debug=opts.extra_verbose)
+    res_tree.OnlyTheseTranslations([self.lang])
+    res_tree.RunGatherers(True)
+    
+    # Dialog IDs are either explicitly listed, or we output all dialogs from the
+    # .grd file
+    dialog_ids = args
+    if not len(dialog_ids):
+      for node in res_tree:
+        if node.name == 'structure' and node.attrs['type'] == 'dialog':
+          dialog_ids.append(node.attrs['name'])
+    
+    self.Process(res_tree, dialog_ids)
+    
+  def Process(self, grd, dialog_ids):
+    '''Outputs an RC file and header file for the dialog 'dialog_id' stored in
+    resource tree 'grd', to self.base_folder, as discussed in this class's
+    documentation.
+    
+    Arguments:
+      grd: grd = grd_reader.Parse(...); grd.RunGatherers()
+      dialog_ids: ['IDD_MYDIALOG', 'IDD_OTHERDIALOG']
+    '''
+    grd.SetOutputContext(self.lang, self.defines)
+    
+    project_name = dialog_ids[0]
+    
+    dir_path = os.path.join(self.base_folder, project_name)
+    if not os.path.isdir(dir_path):
+      os.mkdir(dir_path)
+    
+    # If this fails then we're not on Windows (or you don't have the required
+    # win32all Python libraries installed), so what are you doing mucking
+    # about with RC files anyway? :)
+    import pythoncom
+    
+    # Create the .vcproj file
+    project_text = PROJECT_TEMPLATE.replace(
+      '[[PROJECT_GUID]]', str(pythoncom.CreateGuid())
+      ).replace('[[DIALOG_NAME]]', project_name)
+    fname = os.path.join(dir_path, '%s.vcproj' % project_name)
+    self.WriteFile(fname, project_text)
+    print "Wrote %s" % fname
+    
+    # Create the .rc file
+    # Output all <include> nodes since the dialogs might depend on them (e.g.
+    # for icons and bitmaps).
+    include_items = []
+    for node in grd:
+      if isinstance(node, include.IncludeNode):
+        formatter = node.ItemFormatter('rc_all')
+        if formatter:
+          include_items.append(formatter.Format(node, self.lang))
+    rc_text = RC_TEMPLATE.replace('[[CODEPAGE_NUM]]',
+                                  str(self.codepage_number))
+    rc_text = rc_text.replace('[[INCLUDES]]', ''.join(include_items))
+    
+    # Then output the dialogs we have been asked to output.
+    dialogs = []
+    for dialog_id in dialog_ids:
+      node = grd.GetNodeById(dialog_id)
+      # TODO(joi) Add exception handling for better error reporting
+      formatter = node.ItemFormatter('rc_all')
+      dialogs.append(formatter.Format(node, self.lang))
+    rc_text = rc_text.replace('[[DIALOGS]]', ''.join(dialogs))
+    
+    fname = os.path.join(dir_path, '%s.rc' % project_name)
+    self.WriteFile(fname, rc_text, self.GetEncoding())
+    print "Wrote %s" % fname
+    
+    # Create the resource.h file
+    header_defines = []
+    for node in grd:
+      formatter = node.ItemFormatter('rc_header')
+      if formatter and not isinstance(formatter, rc_header.TopLevel):
+        header_defines.append(formatter.Format(node, self.lang))
+    header_text = HEADER_TEMPLATE.replace('[[DEFINES]]', ''.join(header_defines))
+    fname = os.path.join(dir_path, 'resource.h')
+    self.WriteFile(fname, header_text)
+    print "Wrote %s" % fname
+    
+  def WriteFile(self, filename, contents, encoding='cp1252'):
+    f = util.WrapOutputStream(file(filename, 'wb'), encoding)
+    f.write(contents)
+    f.close()

tools/grit/grit/tool/test.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from grit.tool import interface
+
+class TestTool(interface.Tool):
+  '''This tool does nothing except print out the global options and
+tool-specific arguments that it receives.  It is intended only for testing,
+hence the name :)
+'''
+  
+  def ShortDescription(self):
+    return 'A do-nothing tool for testing command-line parsing.'
+    
+  def Run(self, global_options, my_arguments):
+    print 'NOTE This tool is only for testing the parsing of global options and'
+    print 'tool-specific arguments that it receives.  You may have intended to'
+    print 'run "grit unit" which is the unit-test suite for GRIT.'
+    print 'Options: %s' % repr(global_options)
+    print 'Arguments: %s' % repr(my_arguments)
+    return 0

tools/grit/grit/tool/toolbar_postprocess.py

@@ -0,0 +1,150 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+''' Toolbar postprocessing class. Modifies the previously processed GRD tree
+by creating separate message groups for each of the IDS_COMMAND macros.
+Also adds some identifiers nodes to declare specific ids to be included
+in the generated grh file.
+'''
+
+import sys
+import re
+import postprocess_interface
+import grit.node.empty
+from grit.node import misc
+
+class ToolbarPostProcessor(postprocess_interface.PostProcessor):
+  ''' Defines message groups within the grd file for each of the
+  IDS_COMMAND stuff.
+  '''
+
+  _IDS_COMMAND = re.compile(r'IDS_COMMAND_')
+  _GRAB_PARAMETERS = re.compile(r'(IDS_COMMAND_[a-zA-Z0-9]+)_([a-zA-z0-9]+)')
+
+  def Process(self, rctext, rcpath, grdnode):
+    ''' Processes the data in rctext and grdnode.
+    Args:
+      rctext: string containing the contents of the RC file being processed.
+      rcpath: the path used to access the file.
+      grdnode: the root node of the grd xml data generated by 
+      the rc2grd tool.
+    
+    Return:
+      The root node of the processed GRD tree.
+    '''
+
+    release = grdnode.children[2]
+    messages = release.children[2]
+    
+    identifiers = grit.node.empty.IdentifiersNode()
+    identifiers.StartParsing('identifiers', release)
+    identifiers.EndParsing()
+    release.AddChild(identifiers)
+
+
+    #
+    # Turn the IDS_COMMAND messages into separate message groups
+    # with ids that are offsetted to the message group's first id
+    #
+    previous_name_attr = ''
+    previous_prefix = ''
+    previous_node = ''
+    new_messages_node = self.ConstructNewMessages(release)
+    for node in messages.children[:]:
+      name_attr = node.attrs['name']
+      if self._IDS_COMMAND.search(name_attr):
+        mo = self._GRAB_PARAMETERS.search(name_attr)
+        mp = self._GRAB_PARAMETERS.search(previous_name_attr)
+        if mo and mp:
+          prefix = mo.group(1)
+          previous_prefix = mp.group(1)
+          new_message_id = mp.group(2)
+          if prefix == previous_prefix:
+            messages.RemoveChild(previous_name_attr)
+            previous_node.attrs['offset'] = 'PCI_' + new_message_id
+            del previous_node.attrs['name']
+            new_messages_node.AddChild(previous_node)
+          else:
+            messages.RemoveChild(previous_name_attr)
+            previous_node.attrs['offset'] = 'PCI_' + new_message_id
+            del previous_node.attrs['name']
+            new_messages_node.AddChild(previous_node)
+            new_messages_node.attrs['first_id'] = previous_prefix
+            new_messages_node = self.ConstructNewMessages(release)
+        else:
+          if self._IDS_COMMAND.search(previous_name_attr):
+            messages.RemoveChild(previous_name_attr)
+            previous_prefix = mp.group(1)
+            new_message_id = mp.group(2)
+            previous_node.attrs['offset'] = 'PCI_' + new_message_id
+            del previous_node.attrs['name']
+            new_messages_node.AddChild(previous_node)
+            new_messages_node.attrs['first_id'] = previous_prefix
+            new_messages_node = self.ConstructNewMessages(release)
+      else:
+        if self._IDS_COMMAND.search(previous_name_attr):
+          messages.RemoveChild(previous_name_attr)
+          mp = self._GRAB_PARAMETERS.search(previous_name_attr)
+          previous_prefix = mp.group(1)
+          new_message_id = mp.group(2)
+          previous_node.attrs['offset'] = 'PCI_' + new_message_id
+          del previous_node.attrs['name']
+          new_messages_node.AddChild(previous_node)
+          new_messages_node.attrs['first_id'] = previous_prefix
+          new_messages_node = self.ConstructNewMessages(release)
+      previous_name_attr = name_attr
+      previous_node = node
+
+    
+    self.AddIdentifiers(rctext, identifiers) 
+    return grdnode
+
+  def ConstructNewMessages(self, parent):
+    new_node = grit.node.empty.MessagesNode()
+    new_node.StartParsing('messages', parent)
+    new_node.EndParsing()
+    parent.AddChild(new_node)
+    return new_node
+
+  def AddIdentifiers(self, rctext, node):
+    node.AddChild(misc.IdentifierNode.Construct(node, 'IDS_COMMAND_gcFirst', '12000', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node,
+                                                'IDS_COMMAND_PCI_SPACE', '16', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_BUTTON', '0', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_MENU', '1', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP', '2', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_OPTIONS_TEXT', '3', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_DISABLED', '4', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_MENU', '5', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_MENU_DISABLED', '6', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_OPTIONS', '7', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node, 'PCI_TIP_OPTIONS_DISABLED', '8', ''))
+    node.AddChild(misc.IdentifierNode.Construct(node,
+                                                'PCI_TIP_DISABLED_BY_POLICY', '9', ''))

tools/grit/grit/tool/toolbar_preprocess.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+''' Toolbar preprocessing code. Turns all IDS_COMMAND macros in the RC file
+into simpler constructs that can be understood by GRIT. Also deals with
+expansion of $lf; placeholders into the correct linefeed character.
+'''
+
+import preprocess_interface
+
+import re
+import sys
+import codecs
+
+class ToolbarPreProcessor(preprocess_interface.PreProcessor):
+  ''' Toolbar PreProcessing class.  
+  '''
+
+  _IDS_COMMAND_MACRO = re.compile(r'(.*IDS_COMMAND)\s*\(([a-zA-Z0-9_]*)\s*,\s*([a-zA-Z0-9_]*)\)(.*)')
+  _LINE_FEED_PH = re.compile(r'\$lf;')
+  _PH_COMMENT = re.compile(r'PHRWR')
+  _COMMENT = re.compile(r'^(\s*)//.*')
+
+
+  def Process(self, rctext, rcpath):
+    ''' Processes the data in rctext.
+    Args:
+      rctext: string containing the contents of the RC file being processed
+      rcpath: the path used to access the file.
+    
+    Return:
+      The processed text.
+    '''
+
+    ret = ''
+    rclines = rctext.splitlines()
+    for line in rclines:
+
+      if self._LINE_FEED_PH.search(line):
+        # Replace "$lf;" placeholder comments by an empty line.
+        # this will not be put into the processed result
+        if self._PH_COMMENT.search(line):
+          mm = self._COMMENT.search(line)
+          if mm:
+            line = '%s//' % mm.group(1)
+            
+        else:  
+          # Replace $lf by the right linefeed character
+          line = self._LINE_FEED_PH.sub(r'\\n', line)
+
+      # Deal with IDS_COMMAND_MACRO stuff
+      mo = self._IDS_COMMAND_MACRO.search(line)
+      if mo:
+        line = '%s_%s_%s%s' % (mo.group(1), mo.group(2), mo.group(3), mo.group(4))
+      
+      ret += (line + '\n') 
+
+    return ret
+

tools/grit/grit/tool/transl2tc.py

@@ -0,0 +1,278 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''The 'grit transl2tc' tool.
+'''
+
+
+import getopt
+
+from grit.tool import interface
+from grit.tool import rc2grd
+from grit import grd_reader
+from grit import util
+
+from grit.extern import tclib
+
+
+class TranslationToTc(interface.Tool):
+  '''A tool for importing existing translations in RC format into the
+Translation Console.
+
+Usage:
+
+grit -i GRD transl2tc [-l LIMITS] [RCOPTS] SOURCE_RC TRANSLATED_RC OUT_FILE
+
+The tool needs a "source" RC file, i.e. in English, and an RC file that is a
+translation of precisely the source RC file (not of an older or newer version).
+
+The tool also requires you to provide a .grd file (input file) e.g. using the
+-i global option or the GRIT_INPUT environment variable.  The tool uses
+information from your .grd file to correct placeholder names in the
+translations and ensure that only translatable items and translations still
+being used are output.
+
+This tool will accept all the same RCOPTS as the 'grit rc2grd' tool.  To get
+a list of these options, run 'grit help rc2grd'.
+
+Additionally, you can use the -l option (which must be the first option to the
+tool) to specify a file containing a list of message IDs to which output should
+be limited.  This is only useful if you are limiting the output to your XMB
+files using the 'grit xmb' tool's -l option.  See 'grit help xmb' for how to
+generate a file containing a list of the message IDs in an XMB file.
+
+The tool will scan through both of the RC files as well as any HTML files they
+refer to, and match together the source messages and translated messages.  It
+will output a file (OUTPUT_FILE) you can import directly into the TC using the
+Bulk Translation Upload tool.
+'''
+  
+  def ShortDescription(self):
+    return 'Import existing translations in RC format into the TC'
+  
+  def Setup(self, globopt, args):
+    '''Sets the instance up for use.
+    '''
+    self.SetOptions(globopt)
+    self.rc2grd = rc2grd.Rc2Grd()
+    self.rc2grd.SetOptions(globopt)
+    self.limits = None
+    if len(args) and args[0] == '-l':
+      limit_file = file(args[1])
+      self.limits = limit_file.read().split('\n')
+      limit_file.close()
+      args = args[2:]
+    return self.rc2grd.ParseOptions(args)
+  
+  def Run(self, globopt, args):
+    args = self.Setup(globopt, args)
+    
+    if len(args) != 3:
+      self.Out('This tool takes exactly three arguments:\n'
+             '  1. The path to the original RC file\n'
+             '  2. The path to the translated RC file\n'
+             '  3. The output file path.\n')
+      return 2
+    
+    grd = grd_reader.Parse(self.o.input, debug=self.o.extra_verbose)
+    grd.RunGatherers(recursive = True)
+    
+    source_rc = util.WrapInputStream(file(args[0], 'r'), self.rc2grd.input_encoding)
+    transl_rc = util.WrapInputStream(file(args[1], 'r'), self.rc2grd.input_encoding)
+    translations = self.ExtractTranslations(grd,
+                                            source_rc.read(), args[0],
+                                            transl_rc.read(), args[1])
+    transl_rc.close()
+    source_rc.close()
+
+    output_file = util.WrapOutputStream(file(args[2], 'w'))
+    self.WriteTranslations(output_file, translations.items())
+    output_file.close()
+    
+    self.Out('Wrote output file %s' % args[2])
+    
+  def ExtractTranslations(self, current_grd, source_rc, source_path, transl_rc, transl_path):
+    '''Extracts translations from the translated RC file, matching them with
+    translations in the source RC file to calculate their ID, and correcting
+    placeholders, limiting output to translateables, etc. using the supplied
+    .grd file which is the current .grd file for your project.
+    
+    If this object's 'limits' attribute is not None but a list, the output of
+    this function will be further limited to include only messages that have
+    message IDs in the 'limits' list.
+    
+    Args:
+      current_grd: grit.node.base.Node child, that has had RunGatherers(True) run on it
+      source_rc: Complete text of source RC file
+      source_path: Path to the source RC file
+      transl_rc: Complete text of translated RC file
+      transl_path: Path to the translated RC file
+    
+    Return:
+      { id1 : text1, '12345678' : 'Hello USERNAME, howzit?' }
+    '''
+    source_grd = self.rc2grd.Process(source_rc, source_path)
+    self.VerboseOut('Read %s into GRIT format, running gatherers.\n' % source_path)
+    source_grd.RunGatherers(recursive=True, debug=self.o.extra_verbose)
+    transl_grd = self.rc2grd.Process(transl_rc, transl_path)
+    self.VerboseOut('Read %s into GRIT format, running gatherers.\n' % transl_path)
+    transl_grd.RunGatherers(recursive=True, debug=self.o.extra_verbose)
+    self.VerboseOut('Done running gatherers for %s.\n' % transl_path)
+    
+    # Proceed to create a map from ID to translation, getting the ID from the
+    # source GRD and the translation from the translated GRD.
+    id2transl = {}
+    for source_node in source_grd:
+      source_cliques = source_node.GetCliques()
+      if not len(source_cliques):
+        continue
+      
+      assert 'name' in source_node.attrs, 'All nodes with cliques should have an ID'
+      node_id = source_node.attrs['name']
+      self.ExtraVerboseOut('Processing node %s\n' % node_id)
+      transl_node = transl_grd.GetNodeById(node_id)
+      
+      if transl_node:
+        transl_cliques = transl_node.GetCliques()
+        if not len(transl_cliques) == len(source_cliques):
+          self.Out(
+            'Warning: Translation for %s has wrong # of cliques, skipping.\n' %
+            node_id)
+          continue
+      else:
+        self.Out('Warning: No translation for %s, skipping.\n' % node_id)
+        continue
+      
+      if source_node.name == 'message':
+        # Fixup placeholders as well as possible based on information from
+        # the current .grd file if they are 'TODO_XXXX' placeholders.  We need
+        # to fixup placeholders in the translated message so that it looks right
+        # and we also need to fixup placeholders in the source message so that
+        # its calculated ID will match the current message.
+        current_node = current_grd.GetNodeById(node_id)
+        if current_node:
+          assert len(source_cliques) == 1 and len(current_node.GetCliques()) == 1
+          
+          source_msg = source_cliques[0].GetMessage()
+          current_msg = current_node.GetCliques()[0].GetMessage()
+          
+          # Only do this for messages whose source version has not changed.
+          if (source_msg.GetRealContent() != current_msg.GetRealContent()):
+            self.VerboseOut('Info: Message %s has changed; skipping\n' % node_id)
+          else:
+            transl_msg = transl_cliques[0].GetMessage()
+            transl_content = transl_msg.GetContent()
+            current_content = current_msg.GetContent()
+            source_content = source_msg.GetContent()
+            
+            ok_to_fixup = True
+            if (len(transl_content) != len(current_content)):
+              # message structure of translation is different, don't try fixup
+              ok_to_fixup = False
+            if ok_to_fixup:
+              for ix in range(len(transl_content)):
+                if isinstance(transl_content[ix], tclib.Placeholder):
+                  if not isinstance(current_content[ix], tclib.Placeholder):
+                    ok_to_fixup = False  # structure has changed
+                    break
+                  if (transl_content[ix].GetOriginal() !=
+                      current_content[ix].GetOriginal()):
+                    ok_to_fixup = False  # placeholders have likely been reordered
+                    break
+                else:  # translated part is not a placeholder but a string
+                  if isinstance(current_content[ix], tclib.Placeholder):
+                    ok_to_fixup = False  # placeholders have likely been reordered
+                    break
+            
+            if not ok_to_fixup:
+              self.VerboseOut(
+                'Info: Structure of message %s has changed; skipping.\n' % node_id)
+            else:
+              def Fixup(content, ix):
+                if (isinstance(content[ix], tclib.Placeholder) and
+                    content[ix].GetPresentation().startswith('TODO_')):
+                  assert isinstance(current_content[ix], tclib.Placeholder)
+                  # Get the placeholder ID and example from the current message
+                  content[ix] = current_content[ix]
+              for ix in range(len(transl_content)):
+                Fixup(transl_content, ix)
+                Fixup(source_content, ix)
+      
+      # Only put each translation once into the map.  Warn if translations
+      # for the same message are different.
+      for ix in range(len(transl_cliques)):
+        source_msg = source_cliques[ix].GetMessage()
+        source_msg.GenerateId()  # needed to refresh ID based on new placeholders
+        message_id = source_msg.GetId()
+        translated_content = transl_cliques[ix].GetMessage().GetPresentableContent()
+        
+        if message_id in id2transl:
+          existing_translation = id2transl[message_id]
+          if existing_translation != translated_content:
+            original_text = source_cliques[ix].GetMessage().GetPresentableContent()
+            self.Out('Warning: Two different translations for "%s":\n'
+                   '  Translation 1: "%s"\n'
+                   '  Translation 2: "%s"\n' %
+                   (original_text, existing_translation, translated_content))
+        else:
+          id2transl[message_id] = translated_content
+    
+    # Remove translations for messages that do not occur in the current .grd
+    # or have been marked as not translateable, or do not occur in the 'limits'
+    # list (if it has been set).
+    current_message_ids = current_grd.UberClique().AllMessageIds()
+    for message_id in id2transl.keys():
+      if (message_id not in current_message_ids or
+          not current_grd.UberClique().BestClique(message_id).IsTranslateable() or
+          (self.limits and message_id not in self.limits)):
+        del id2transl[message_id]
+    
+    return id2transl
+  
+  # static method
+  def WriteTranslations(output_file, translations):
+    '''Writes the provided list of translations to the provided output file
+    in the format used by the TC's Bulk Translation Upload tool.  The file
+    must be UTF-8 encoded.
+    
+    Args:
+      output_file: util.WrapOutputStream(file('bingo.out', 'w'))
+      translations: [ [id1, text1], ['12345678', 'Hello USERNAME, howzit?'] ]
+    
+    Return:
+      None
+    '''
+    for id, text in translations:
+      text = text.replace('<', '&lt;').replace('>', '&gt;')
+      output_file.write(id)
+      output_file.write(' ')
+      output_file.write(text)
+      output_file.write('\n')
+  WriteTranslations = staticmethod(WriteTranslations)

tools/grit/grit/tool/transl2tc_unittest.py

@@ -0,0 +1,155 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for the 'grit transl2tc' tool.'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+
+import StringIO
+import unittest
+
+from grit.tool import transl2tc
+from grit import grd_reader
+from grit import util
+
+
+def MakeOptions():
+  from grit import grit_runner
+  return grit_runner.Options()
+
+
+class TranslationToTcUnittest(unittest.TestCase):
+  
+  def testOutput(self):
+    buf = StringIO.StringIO()
+    tool = transl2tc.TranslationToTc()
+    translations = [
+      ['1', 'Hello USERNAME, how are you?'],
+      ['12', 'Howdie doodie!'],
+      ['123', 'Hello\n\nthere\n\nhow are you?'],
+      ['1234', 'Hello is > goodbye but < howdie pardner'],
+    ]
+    tool.WriteTranslations(buf, translations)
+    output = buf.getvalue()
+    self.failUnless(output.strip() == '''
+1 Hello USERNAME, how are you?
+12 Howdie doodie!
+123 Hello
+
+there
+
+how are you?
+1234 Hello is &gt; goodbye but &lt; howdie pardner
+'''.strip())
+
+  def testExtractTranslations(self):
+    path = util.PathFromRoot('grit/test/data')
+    current_grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <grit latest_public_release="2" source_lang_id="en-US" current_release="3" base_dir=".">
+        <release seq="3">
+          <messages>
+            <message name="IDS_SIMPLE">
+              One
+            </message>
+            <message name="IDS_PLACEHOLDER">
+              <ph name="NUMBIRDS">%s<ex>3</ex></ph> birds
+            </message>
+            <message name="IDS_PLACEHOLDERS">
+              <ph name="ITEM">%d<ex>1</ex></ph> of <ph name="COUNT">%d<ex>3</ex></ph>
+            </message>
+            <message name="IDS_REORDERED_PLACEHOLDERS">
+              <ph name="ITEM">$1<ex>1</ex></ph> of <ph name="COUNT">$2<ex>3</ex></ph>
+            </message>
+            <message name="IDS_CHANGED">
+              This is the new version
+            </message>
+            <message name="IDS_TWIN_1">Hello</message>
+            <message name="IDS_TWIN_2">Hello</message>
+            <message name="IDS_NOT_TRANSLATEABLE" translateable="false">:</message>
+            <message name="IDS_LONGER_TRANSLATED">
+              Removed document <ph name="FILENAME">$1<ex>c:\temp</ex></ph>
+            </message>
+            <message name="IDS_DIFFERENT_TWIN_1">Howdie</message>
+            <message name="IDS_DIFFERENT_TWIN_2">Howdie</message>
+          </messages>
+          <structures>
+            <structure type="dialog" name="IDD_ABOUTBOX" encoding="utf-16" file="klonk.rc" />
+            <structure type="menu" name="IDC_KLONKMENU" encoding="utf-16" file="klonk.rc" />
+          </structures>
+        </release>
+      </grit>'''), path)
+    current_grd.RunGatherers(recursive=True)
+    
+    source_rc_path = util.PathFromRoot('grit/test/data/source.rc')
+    source_rc = file(source_rc_path).read()
+    transl_rc_path = util.PathFromRoot('grit/test/data/transl.rc')
+    transl_rc = file(transl_rc_path).read()
+    
+    tool = transl2tc.TranslationToTc()
+    output_buf = StringIO.StringIO()
+    globopts = MakeOptions()
+    globopts.verbose = True
+    globopts.output_stream = output_buf
+    tool.Setup(globopts, [])
+    translations = tool.ExtractTranslations(current_grd,
+                                            source_rc, source_rc_path,
+                                            transl_rc, transl_rc_path)
+    
+    values = translations.values()
+    output = output_buf.getvalue()
+    
+    self.failUnless('Ein' in values)
+    self.failUnless('NUMBIRDS Vogeln' in values)
+    self.failUnless('ITEM von COUNT' in values)
+    self.failUnless(values.count('Hallo') == 1)
+    self.failIf('Dass war die alte Version' in values)
+    self.failIf(':' in values)
+    self.failIf('Dokument FILENAME ist entfernt worden' in values)
+    self.failIf('Nicht verwendet' in values)
+    self.failUnless(('Howdie' in values or 'Hallo sagt man' in values) and not
+      ('Howdie' in values and 'Hallo sagt man' in values))
+    
+    self.failUnless('XX01XX&SkraXX02XX&HaettaXX03XXThetta er "Klonk" sem eg fylaXX04XXgonkurinnXX05XXKlonk && er "gott"XX06XX&HjalpXX07XX&Um...XX08XX' in values)
+    
+    self.failUnless('I lagi' in values)
+    
+    self.failUnless(output.count('Structure of message IDS_REORDERED_PLACEHOLDERS has changed'))
+    self.failUnless(output.count('Message IDS_CHANGED has changed'))
+    self.failUnless(output.count('Structure of message IDS_LONGER_TRANSLATED has changed'))
+    self.failUnless(output.count('Two different translations for "Howdie"'))
+    self.failUnless(output.count('IDD_DIFFERENT_LENGTH_IN_TRANSL has wrong # of cliques'))
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/tool/unit.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''GRIT tool that runs the unit test suite for GRIT.'''
+
+
+import unittest
+
+import grit.test_suite_all
+from grit.tool import interface
+
+
+class UnitTestTool(interface.Tool):
+  '''By using this tool (e.g. 'grit unit') you run all the unit tests for GRIT.
+This happens in the environment that is set up by the basic GRIT runner, i.e.
+whether to run disconnected has been specified, etc.'''
+  
+  def ShortDescription(self):
+    return 'Use this tool to run all the unit tests for GRIT.'
+  
+  def Run(self, opts, args):
+    return unittest.TextTestRunner(verbosity=2).run(
+      grit.test_suite_all.TestSuiteAll())

tools/grit/grit/util.py

@@ -0,0 +1,334 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Utilities used by GRIT.
+'''
+
+import sys
+import os.path
+import codecs
+import htmlentitydefs
+import re
+import time
+from xml.sax import saxutils
+
+_root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+
+
+# Matches all of the resource IDs predefined by Windows.
+# The '\b' before and after each word makes sure these match only whole words and
+# not the beginning of any word.. eg. ID_FILE_NEW will not match ID_FILE_NEW_PROJECT
+# see http://www.amk.ca/python/howto/regex/ (search for "\bclass\b" inside the html page)
+SYSTEM_IDENTIFIERS = re.compile(
+  r'''\bIDOK\b | \bIDCANCEL\b | \bIDC_STATIC\b | \bIDYES\b | \bIDNO\b |
+      \bID_FILE_NEW\b | \bID_FILE_OPEN\b | \bID_FILE_CLOSE\b | \bID_FILE_SAVE\b |
+      \bID_FILE_SAVE_AS\b | \bID_FILE_PAGE_SETUP\b | \bID_FILE_PRINT_SETUP\b | 
+      \bID_FILE_PRINT\b | \bID_FILE_PRINT_DIRECT\b | \bID_FILE_PRINT_PREVIEW\b | 
+      \bID_FILE_UPDATE\b | \bID_FILE_SAVE_COPY_AS\b | \bID_FILE_SEND_MAIL\b | 
+      \bID_FILE_MRU_FIRST\b | \bID_FILE_MRU_LAST\b |
+      \bID_EDIT_CLEAR\b | \bID_EDIT_CLEAR_ALL\b | \bID_EDIT_COPY\b |
+      \bID_EDIT_CUT\b | \bID_EDIT_FIND\b | \bID_EDIT_PASTE\b | \bID_EDIT_PASTE_LINK\b |
+      \bID_EDIT_PASTE_SPECIAL\b | \bID_EDIT_REPEAT\b | \bID_EDIT_REPLACE\b |
+      \bID_EDIT_SELECT_ALL\b | \bID_EDIT_UNDO\b | \bID_EDIT_REDO\b |
+      \bVS_VERSION_INFO\b | \bIDRETRY''', re.VERBOSE);
+
+
+# Matches character entities, whether specified by name, decimal or hex.
+_HTML_ENTITY = re.compile(
+  '&(#(?P<decimal>[0-9]+)|#x(?P<hex>[a-fA-F0-9]+)|(?P<named>[a-z0-9]+));',
+  re.IGNORECASE)
+
+# Matches characters that should be HTML-escaped.  This is <, > and &, but only
+# if the & is not the start of an HTML character entity.
+_HTML_CHARS_TO_ESCAPE = re.compile('"|<|>|&(?!#[0-9]+|#x[0-9a-z]+|[a-z]+;)',
+                                   re.IGNORECASE | re.MULTILINE)
+
+
+def WrapInputStream(stream, encoding = 'utf-8'):
+  '''Returns a stream that wraps the provided stream, making it read characters
+  using the specified encoding.'''
+  (e, d, sr, sw) = codecs.lookup(encoding)
+  return sr(stream)
+
+
+def WrapOutputStream(stream, encoding = 'utf-8'):
+  '''Returns a stream that wraps the provided stream, making it write
+  characters using the specified encoding.'''
+  (e, d, sr, sw) = codecs.lookup(encoding)
+  return sw(stream)
+
+
+def ChangeStdoutEncoding(encoding = 'utf-8'):
+  '''Changes STDOUT to print characters using the specified encoding.'''
+  sys.stdout = WrapOutputStream(sys.stdout, encoding)
+
+
+def EscapeHtml(text, escape_quotes = False):
+  '''Returns 'text' with <, > and & (and optionally ") escaped to named HTML
+  entities.  Any existing named entity or HTML entity defined by decimal or
+  hex code will be left untouched.  This is appropriate for escaping text for
+  inclusion in HTML, but not for XML.
+  '''
+  def Replace(match):
+    if match.group() == '&': return '&amp;'
+    elif match.group() == '<': return '&lt;'
+    elif match.group() == '>': return '&gt;'
+    elif match.group() == '"':
+      if escape_quotes: return '&quot;'
+      else: return match.group()
+    else: assert False
+  out = _HTML_CHARS_TO_ESCAPE.sub(Replace, text)
+  return out
+
+
+def UnescapeHtml(text, replace_nbsp=True):
+  '''Returns 'text' with all HTML character entities (both named character
+  entities and those specified by decimal or hexadecimal Unicode ordinal)
+  replaced by their Unicode characters (or latin1 characters if possible).
+  
+  The only exception is that &nbsp; will not be escaped if 'replace_nbsp' is
+  False.
+  '''
+  def Replace(match):
+    groups = match.groupdict()
+    if groups['hex']:
+      return unichr(int(groups['hex'], 16))
+    elif groups['decimal']:
+      return unichr(int(groups['decimal'], 10))
+    else:
+      name = groups['named']
+      if name == 'nbsp' and not replace_nbsp:
+        return match.group()  # Don't replace &nbsp;
+      assert name != None
+      if name in htmlentitydefs.name2codepoint.keys():
+        return unichr(htmlentitydefs.name2codepoint[name])
+      else:
+        return match.group()  # Unknown HTML character entity - don't replace
+  
+  out = _HTML_ENTITY.sub(Replace, text)
+  return out
+
+
+def EncodeCdata(cdata):
+  '''Returns the provided cdata in either escaped format or <![CDATA[xxx]]>
+  format, depending on which is more appropriate for easy editing.  The data
+  is escaped for inclusion in an XML element's body.
+  
+  Args:
+    cdata: 'If x < y and y < z then x < z'
+  
+  Return:
+    '<![CDATA[If x < y and y < z then x < z]]>'
+  '''
+  if cdata.count('<') > 1 or cdata.count('>') > 1 and cdata.count(']]>') == 0:
+    return '<![CDATA[%s]]>' % cdata
+  else:
+    return saxutils.escape(cdata)
+
+
+def FixupNamedParam(function, param_name, param_value):
+  '''Returns a closure that is identical to 'function' but ensures that the
+  named parameter 'param_name' is always set to 'param_value' unless explicitly
+  set by the caller.
+  
+  Args:
+    function: callable
+    param_name: 'bingo'
+    param_value: 'bongo' (any type)
+  
+  Return:
+    callable
+  '''
+  def FixupClosure(*args, **kw):
+    if not param_name in kw:
+      kw[param_name] = param_value
+    return function(*args, **kw)
+  return FixupClosure
+
+
+def PathFromRoot(path):
+  '''Takes a path relative to the root directory for GRIT (the one that grit.py
+  resides in) and returns a path that is either absolute or relative to the
+  current working directory (i.e .a path you can use to open the file).
+  
+  Args:
+    path: 'rel_dir\file.ext'
+  
+  Return:
+    'c:\src\tools\rel_dir\file.ext
+  '''
+  return os.path.normpath(os.path.join(_root_dir, path))
+
+
+def FixRootForUnittest(root_node, dir=PathFromRoot('.')):
+  '''Adds a GetBaseDir() method to 'root_node', making unittesting easier.'''
+  def GetBaseDir():
+    '''Returns a fake base directory.'''
+    return dir
+  def GetSourceLanguage():
+    return 'en'
+  if not hasattr(root_node, 'GetBaseDir'):
+    setattr(root_node, 'GetBaseDir', GetBaseDir)
+    setattr(root_node, 'GetSourceLanguage', GetSourceLanguage)
+
+
+def dirname(filename):
+  '''Version of os.path.dirname() that never returns empty paths (returns
+  '.' if the result of os.path.dirname() is empty).
+  '''
+  ret = os.path.dirname(filename)
+  if ret == '':
+    ret = '.'
+  return ret
+
+
+def normpath(path):
+  '''Version of os.path.normpath that also changes backward slashes to
+  forward slashes when not running on Windows.
+  '''
+  # This is safe to always do because the Windows version of os.path.normpath
+  # will replace forward slashes with backward slashes.
+  path = path.replace('\\', '/')
+  return os.path.normpath(path)
+
+
+_LANGUAGE_SPLIT_RE = re.compile('-|_|/')
+
+
+def CanonicalLanguage(code):
+  '''Canonicalizes two-part language codes by using a dash and making the
+  second part upper case.  Returns one-part language codes unchanged.
+  
+  Args:
+    code: 'zh_cn'
+  
+  Return:
+    code: 'zh-CN'
+  '''
+  parts = _LANGUAGE_SPLIT_RE.split(code)
+  code = [ parts[0] ]
+  for part in parts[1:]:
+    code.append(part.upper())
+  return '-'.join(code)
+
+
+_LANG_TO_CODEPAGE = {
+  'en' : 1252,
+  'fr' : 1252,
+  'it' : 1252,
+  'de' : 1252,
+  'es' : 1252,
+  'nl' : 1252,
+  'sv' : 1252,
+  'no' : 1252,
+  'da' : 1252,
+  'fi' : 1252,
+  'pt-BR' : 1252,
+  'ru' : 1251,
+  'ja' : 932,
+  'zh-TW' : 950,
+  'zh-CN' : 936,
+  'ko' : 949,
+}
+
+
+def LanguageToCodepage(lang):
+  '''Returns the codepage _number_ that can be used to represent 'lang', which
+  may be either in formats such as 'en', 'pt_br', 'pt-BR', etc.
+  
+  The codepage returned will be one of the 'cpXXXX' codepage numbers.
+  
+  Args:
+    lang: 'de'
+  
+  Return:
+    1252
+  '''
+  lang = CanonicalLanguage(lang)
+  if lang in _LANG_TO_CODEPAGE:
+    return _LANG_TO_CODEPAGE[lang]
+  else:
+    print "Not sure which codepage to use for %s, assuming cp1252" % lang
+    return 1252
+
+def NewClassInstance(class_name, class_type):
+  '''Returns an instance of the class specified in classname
+
+  Args:
+    class_name: the fully qualified, dot separated package + classname,
+    i.e. "my.package.name.MyClass". Short class names are not supported.
+    class_type: the class or superclass this object must implement
+
+  Return:
+    An instance of the class, or None if none was found
+  '''
+  lastdot = class_name.rfind('.')
+  module_name = ''
+  if lastdot >= 0:  
+    module_name = class_name[0:lastdot]
+    if module_name:
+      class_name = class_name[lastdot+1:]
+      module = __import__(module_name, globals(), locals(), [''])
+      if hasattr(module, class_name):
+        class_ = getattr(module, class_name)
+        class_instance = class_()
+        if isinstance(class_instance, class_type):
+          return class_instance
+  return None 
+
+
+def FixLineEnd(text, line_end):
+  # First normalize
+  text = text.replace('\r\n', '\n')
+  text = text.replace('\r', '\n')
+  # Then fix
+  text = text.replace('\n', line_end)
+  return text
+
+
+def BoolToString(bool):
+  if bool:
+    return 'true'
+  else:
+    return 'false'
+  
+
+verbose = False
+extra_verbose = False
+
+def IsVerbose():
+  return verbose
+
+def IsExtraVerbose():
+  return extra_verbose
+
+def GetCurrentYear():
+  '''Returns the current 4-digit year as an integer.'''
+  return time.localtime()[0]

tools/grit/grit/util_unittest.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit test that checks some of util functions.
+'''
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import unittest
+
+from grit import util
+
+
+class UtilUnittest(unittest.TestCase):
+  ''' Tests functions from util
+  '''
+  
+  def testNewClassInstance(self):
+    # Test short class name with no fully qualified package name
+    # Should fail, it is not supported by the function now (as documented)
+    cls = util.NewClassInstance('grit.util.TestClassToLoad',
+                                TestBaseClassToLoad)
+    self.failUnless(cls == None)
+  
+    # Test non existent class name
+    cls = util.NewClassInstance('grit.util_unittest.NotExistingClass',
+                                TestBaseClassToLoad) 
+    self.failUnless(cls == None)
+
+    # Test valid class name and valid base class
+    cls = util.NewClassInstance('grit.util_unittest.TestClassToLoad',
+                                TestBaseClassToLoad)
+    self.failUnless(isinstance(cls, TestBaseClassToLoad))
+
+    # Test valid class name with wrong hierarchy
+    cls = util.NewClassInstance('grit.util_unittest.TestClassNoBase',
+                                TestBaseClassToLoad)
+    self.failUnless(cls == None)
+  
+  def testCanonicalLanguage(self):
+    self.failUnless(util.CanonicalLanguage('en') == 'en')
+    self.failUnless(util.CanonicalLanguage('pt_br') == 'pt-BR')
+    self.failUnless(util.CanonicalLanguage('pt-br') == 'pt-BR')
+    self.failUnless(util.CanonicalLanguage('pt-BR') == 'pt-BR')
+    self.failUnless(util.CanonicalLanguage('pt/br') == 'pt-BR')
+    self.failUnless(util.CanonicalLanguage('pt/BR') == 'pt-BR')
+    self.failUnless(util.CanonicalLanguage('no_no_bokmal') == 'no-NO-BOKMAL')
+
+  def testUnescapeHtml(self):
+    self.failUnless(util.UnescapeHtml('ϲ') == unichr(1010))
+    self.failUnless(util.UnescapeHtml('ꯍ') == unichr(43981))
+
+class TestBaseClassToLoad(object):
+  pass
+
+class TestClassToLoad(TestBaseClassToLoad):
+  pass
+
+class TestClassNoBase(object):
+  pass
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/xtb_reader.py

@@ -0,0 +1,123 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Fast and efficient parser for XTB files.
+'''
+
+
+import xml.sax
+import xml.sax.handler
+
+
+class XtbContentHandler(xml.sax.handler.ContentHandler):
+  '''A content handler that calls a given callback function for each
+  translation in the XTB file.
+  '''
+  
+  def __init__(self, callback, debug=False):
+    self.callback = callback
+    self.debug = debug
+    # 0 if we are not currently parsing a translation, otherwise the message
+    # ID of that translation.
+    self.current_id = 0
+    # Empty if we are not currently parsing a translation, otherwise the
+    # parts we have for that translation - a list of tuples
+    # (is_placeholder, text)
+    self.current_structure = []
+    # Set to the language ID when we see the <translationbundle> node.
+    self.language = ''
+  
+  def startElement(self, name, attrs):
+    if name == 'translation':
+      assert (self.current_id == 0 and len(self.current_structure) == 0,
+              "Didn't expect a <translation> element here.")
+      self.current_id = attrs.getValue('id')
+    elif name == 'ph':
+      assert self.current_id != 0, "Didn't expect a <ph> element here."
+      self.current_structure.append((True, attrs.getValue('name')))
+    elif name == 'translationbundle':
+      self.language = attrs.getValue('lang')
+
+  def endElement(self, name):
+    if name == 'translation':
+      assert self.current_id != 0
+      self.callback(self.current_id, self.current_structure)
+      self.current_id = 0
+      self.current_structure = []
+
+  def characters(self, content):
+    if self.current_id != 0:
+      # We are inside a <translation> node so just add the characters to our
+      # structure.
+      #
+      # This naive way of handling characters is OK because in the XTB format,
+      # <ph> nodes are always empty (always <ph name="XXX"/>) and whitespace
+      # inside the <translation> node should be preserved.
+      self.current_structure.append((False, content))
+
+
+class XtbErrorHandler(xml.sax.handler.ErrorHandler):
+  def error(self, exception):
+    pass
+  
+  def fatalError(self, exception):
+    raise exception
+  
+  def warning(self, exception):
+    pass
+
+
+def Parse(xtb_file, callback_function, debug=False):
+  '''Parse xtb_file, making a call to callback_function for every translation
+  in the XTB file.
+  
+  The callback function must have the signature as described below.  The 'parts'
+  parameter is a list of tuples (is_placeholder, text).  The 'text' part is
+  either the raw text (if is_placeholder is False) or the name of the placeholder
+  (if is_placeholder is True).
+  
+  Args:
+    xtb_file:           file('fr.xtb')
+    callback_function:  def Callback(msg_id, parts): pass
+  
+  Return:
+    The language of the XTB, e.g. 'fr'
+  '''
+  # Start by advancing the file pointer past the DOCTYPE thing, as the TC
+  # uses a path to the DTD that only works in Unix.
+  # TODO(joi) Remove this ugly hack by getting the TC gang to change the
+  # XTB files somehow?
+  front_of_file = xtb_file.read(1024)
+  xtb_file.seek(front_of_file.find('<translationbundle'))
+  
+  handler = XtbContentHandler(callback=callback_function, debug=debug)
+  xml.sax.parse(xtb_file, handler)
+  assert handler.language != ''
+  return handler.language

tools/grit/grit/xtb_reader_unittest.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python2.4
+# Copyright 2008, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Unit tests for grit.xtb_reader'''
+
+
+import os
+import sys
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+
+import StringIO
+import unittest
+
+from grit import xtb_reader
+from grit import clique
+from grit import grd_reader
+from grit import tclib
+from grit import util
+
+
+class XtbReaderUnittest(unittest.TestCase):
+  def testParsing(self):
+    xtb_file = StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <!DOCTYPE translationbundle>
+      <translationbundle lang="fr">
+        <translation id="5282608565720904145">Bingo.</translation>
+        <translation id="2955977306445326147">Bongo longo.</translation>
+        <translation id="238824332917605038">Hullo</translation>
+        <translation id="6629135689895381486"><ph name="PROBLEM_REPORT"/> peut <ph name="START_LINK"/>utilisation excessive de majuscules<ph name="END_LINK"/>.</translation>
+        <translation id="7729135689895381486">Hello
+this is another line
+and another
+
+and another after a blank line.</translation>
+      </translationbundle>''')
+    
+    messages = []
+    def Callback(id, structure):
+      messages.append((id, structure))
+    xtb_reader.Parse(xtb_file, Callback)
+    self.failUnless(len(messages[0][1]) == 1)
+    self.failUnless(messages[3][1][0])  # PROBLEM_REPORT placeholder
+    self.failUnless(messages[4][0] == '7729135689895381486')
+    self.failUnless(messages[4][1][7][1] == 'and another after a blank line.')
+
+  def testParsingIntoMessages(self):
+    grd = grd_reader.Parse(StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <messages>
+        <message name="ID_MEGA">Fantastic!</message>
+        <message name="ID_HELLO_USER">Hello <ph name="USERNAME">%s<ex>Joi</ex></ph></message>
+      </messages>'''), dir='.', flexible_root=True)
+    
+    clique_mega = grd.children[0].GetCliques()[0]
+    msg_mega = clique_mega.GetMessage()
+    clique_hello_user = grd.children[1].GetCliques()[0]
+    msg_hello_user = clique_hello_user.GetMessage()
+    
+    xtb_file = StringIO.StringIO('''<?xml version="1.0" encoding="UTF-8"?>
+      <!DOCTYPE translationbundle>
+      <translationbundle lang="is">
+        <translation id="%s">Meirihattar!</translation>
+        <translation id="%s">Saelir <ph name="USERNAME"/></translation>
+      </translationbundle>''' % (msg_mega.GetId(), msg_hello_user.GetId()))
+    
+    xtb_reader.Parse(xtb_file, grd.UberClique().GenerateXtbParserCallback('is'))
+    self.failUnless(clique_mega.MessageForLanguage('is').GetRealContent() ==
+                    'Meirihattar!')
+    self.failUnless(clique_hello_user.MessageForLanguage('is').GetRealContent() ==
+                    'Saelir %s')
+
+  def testParseLargeFile(self):
+    def Callback(id, structure):
+      pass
+    xtb = file(util.PathFromRoot('grit/test/data/fr.xtb'))
+    xtb_reader.Parse(xtb, Callback)
+    xtb.close()
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/measure_page_load_time/ff_ext/chrome.manifest

@@ -0,0 +1,2 @@
+content	measurepageloadtimeextension	content/
+overlay	chrome://browser/content/browser.xul	chrome://measurepageloadtimeextension/content/firefoxOverlay.xul

tools/measure_page_load_time/ff_ext/content/firefoxOverlay.xul

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet href="chrome://measurepageloadtimeextension/skin/overlay.css" type="text/css"?>
+<!DOCTYPE overlay SYSTEM "chrome://measurepageloadtimeextension/locale/measurepageloadtimeextension.dtd">
+<overlay id="measurepageloadtimeextension-overlay"
+         xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+  <script src="measure_page_load_time.js"/>
+</overlay>

tools/measure_page_load_time/ff_ext/content/measure_page_load_time.js

@@ -0,0 +1,207 @@
+// Copyright 2008 Google Inc.
+// All Rights Reserved.
+/**
+ * @fileoverview measure_page_load_time.js implements a Firefox extension
+ * for measuring how long a page takes to load. It waits on TCP port
+ * 42492 for connections, then accepts URLs and returns strings of the
+ * form url,time, where "time" is the load time in milliseconds or the
+ * string "timeout" or "error". Load time is measured from the call to
+ * loadURI until the load event fires, or until the status changes to
+ * STATUS_STOP if the load event doesn't fire (there's an error.)
+ * @author jhaas@google.com (Jonathan Haas) */
+
+// Shorthand reference to nsIWebProgress[Listener] interfaces
+var IWP  = Components.interfaces.nsIWebProgress;
+var IWPL = Components.interfaces.nsIWebProgressListener;
+    
+
+var MPLT = {
+  /**
+   * Constants
+   */
+  PORT_NUMBER : 42492,      // port to listen for connections on
+  TIME_OUT : 4 * 60 * 1000, // timeout in 4 minutes
+
+  /**
+   * Incoming URL buffer
+   * @type {string}
+   */
+  textBuffer : '',
+  
+  /**
+   * URL we're currently visiting
+   * @type {string}
+   */
+  URL : '',
+  
+  /**
+   * Listener to accept incoming connections
+   * @type {nsIServerSocketListener}
+   */
+  acceptListener :
+  {
+    onSocketAccepted : function(serverSocket, transport)
+    {
+      MPLT.streamInput  = transport.openInputStream(0,0,0);
+      MPLT.streamOutput = transport.openOutputStream(0,0,0);
+      
+      MPLT.scriptStream = Components.classes['@mozilla.org/scriptableinputstream;1']
+        .createInstance(Components.interfaces.nsIScriptableInputStream);
+      MPLT.scriptStream.init(MPLT.streamInput);
+      MPLT.pump = Components.classes['@mozilla.org/network/input-stream-pump;1']
+        .createInstance(Components.interfaces.nsIInputStreamPump);
+      MPLT.pump.init(MPLT.streamInput, -1, -1, 0, 0, false);
+      MPLT.pump.asyncRead(MPLT.dataListener,null);    
+    },
+
+    onStopListening : function(){}
+  },
+
+  /**
+   * Listener for network input
+   * @type {nsIStreamListener}
+   */
+  dataListener : 
+  {
+    onStartRequest: function(){},
+    onStopRequest: function(){},
+    onDataAvailable: function(request, context, inputStream, offset, count){
+      // Add the received data to the buffer, then process it
+      // Change CRLF to newline while we're at it
+      MPLT.textBuffer += MPLT.scriptStream.read(count).replace('\r\n', '\n');
+      
+      MPLT.process();
+    }
+  },
+  
+  /**
+   * Process the incoming data buffer
+   */
+  process : function()
+  {
+    // If we're waiting for a page to finish loading, wait
+    if (MPLT.timeLoadStarted)
+      return;
+    
+    // Look for a carriage return
+    var firstCR = MPLT.textBuffer.indexOf('\n');
+    
+    // If we haven't received a carriage return yet, wait
+    if (firstCR < 0) 
+      return;
+      
+    // If the first character was a carriage return, we're done!
+    if (firstCR == 0) {
+      MPLT.textBuffer = '';
+      MPLT.streamInput.close();
+      MPLT.streamOutput.close();
+      
+      return;
+    }
+    
+    // Remove the URL from the buffer
+    MPLT.URL = MPLT.textBuffer.substr(0, firstCR);
+    MPLT.textBuffer = MPLT.textBuffer.substr(firstCR + 1);
+
+    // Remember the current time and navigate to the new URL    
+    MPLT.timeLoadStarted = new Date();
+    gBrowser.loadURIWithFlags(MPLT.URL, gBrowser.LOAD_FLAGS_BYPASS_CACHE);
+    setTimeout('MPLT.onTimeOut()', MPLT.TIME_OUT);
+  },
+  
+  /**
+   * Page load completion handler
+   */
+  onPageLoad : function(e) {
+    // Ignore loads of non-HTML documents
+    if (!(e.originalTarget instanceof HTMLDocument))
+      return;
+    
+    // Also ignore subframe loads
+    if (e.originalTarget.defaultView.frameElement)
+      return;
+      
+    clearTimeout();
+    var timeElapsed = new Date() - MPLT.timeLoadStarted;
+    
+    MPLT.outputResult(timeElapsed);
+  },
+  
+  /**
+   * Timeout handler
+   */
+  onTimeOut : function() {
+    gBrowser.stop();
+    
+    MPLT.outputResult('timeout');
+  },
+  
+  
+  /**
+   * Sends a properly-formatted result to the client
+   * @param {string} result  The value to send along with the URL
+   */
+  outputResult : function(result) {
+
+    if (MPLT.URL) {
+      var outputString = MPLT.URL + ',' + result + '\n';
+      MPLT.streamOutput.write(outputString, outputString.length);
+      MPLT.URL = '';
+    }
+    
+    MPLT.timeLoadStarted = null;
+    MPLT.process();
+  },
+
+  /**
+   * Time the page load started. If null, we're waiting for the
+   * initial page load, or otherwise don't care about the page
+   * that's currently loading
+   * @type {number}
+   */
+  timeLoadStarted : null,
+
+  /*
+   * TODO(jhaas): add support for nsIWebProgressListener
+   * If the URL being visited died as part of a network error
+   * (host not found, connection reset by peer, etc), the onload
+   * event doesn't fire. The only way to catch it would be in
+   * a web progress listener. However, nsIWebProgress is not
+   * behaving according to documentation. More research is needed.
+   * For now, omitting it means that if any of our URLs are "dirty" 
+   * (do not point to real web servers with real responses), we'll log 
+   * them as timeouts. This doesn't affect pages where the server
+   * exists but returns an error code.
+   */
+     
+  /**
+   * Initialize the plugin, create the socket and listen
+   */
+  initialize: function() {
+    // Register for page load events
+    gBrowser.addEventListener('load', this.onPageLoad, true);
+    
+    // Set a timeout to wait for the initial page to load
+    MPLT.timeLoadStarted = new Date();
+    setTimeout('MPLT.onTimeOut()', MPLT.TIME_OUT);
+    
+    // Create the listening socket
+    MPLT.serverSocket = Components.classes['@mozilla.org/network/server-socket;1']
+                         .createInstance(Components.interfaces.nsIServerSocket);
+
+    MPLT.serverSocket.init(MPLT.PORT_NUMBER, true, 1);
+    MPLT.serverSocket.asyncListen(this.acceptListener);  
+  },
+
+  /**
+   * Close the socket(s)
+   */
+  deinitialize: function() {
+    if (MPLT.streamInput)  MPLT.streamInput.close();
+    if (MPLT.streamOutput) MPLT.streamOutput.close();
+    if (MPLT.serverSocket) MPLT.serverSocket.close();
+  }
+};
+
+window.addEventListener('load', function(e) { MPLT.initialize(); }, false);
+window.addEventListener('unload', function(e) { MPLT.deinitialize(); }, false);

tools/measure_page_load_time/ff_ext/install.rdf

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" 
+ xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>measurepageloadtimeextension@google.com</em:id>
+    <em:name>MeasurePageLoadTime</em:name>
+    <em:version>1.0</em:version>
+    <em:creator>Jonathan Haas</em:creator>
+    <em:targetApplication>
+      <Description>
+        <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id> <!-- firefox -->
+        <em:minVersion>1.5</em:minVersion>
+        <em:maxVersion>2.0.0.*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+  </Description>
+</RDF>

tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.cpp

@@ -0,0 +1,97 @@
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// MeasurePageLoadTime.cpp : Implementation of DLL Exports.
+
+#include "stdafx.h"
+#include "resource.h"
+#include "MeasurePageLoadTime.h"
+
+
+class CMeasurePageLoadTimeModule : public CAtlDllModuleT< CMeasurePageLoadTimeModule >
+{
+public :
+	DECLARE_LIBID(LIBID_MeasurePageLoadTimeLib)
+	DECLARE_REGISTRY_APPID_RESOURCEID(IDR_MEASUREPAGELOADTIME, "{56C6D9F9-643C-4F6E-906C-5F7CECB23C24}")
+};
+
+CMeasurePageLoadTimeModule _AtlModule;
+
+
+#ifdef _MANAGED
+#pragma managed(push, off)
+#endif
+
+// DLL Entry Point
+extern "C" BOOL WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)
+{
+    if (dwReason == DLL_PROCESS_ATTACH)
+    {
+        DisableThreadLibraryCalls(hInstance);
+    }
+    return _AtlModule.DllMain(dwReason, lpReserved); 
+}
+
+#ifdef _MANAGED
+#pragma managed(pop)
+#endif
+
+
+
+
+// Used to determine whether the DLL can be unloaded by OLE
+STDAPI DllCanUnloadNow(void)
+{
+    return _AtlModule.DllCanUnloadNow();
+}
+
+
+// Returns a class factory to create an object of the requested type
+STDAPI DllGetClassObject(REFCLSID rclsid, REFIID riid, LPVOID* ppv)
+{
+    return _AtlModule.DllGetClassObject(rclsid, riid, ppv);
+}
+
+
+// DllRegisterServer - Adds entries to the system registry
+STDAPI DllRegisterServer(void)
+{
+    // registers object, typelib and all interfaces in typelib
+    HRESULT hr = _AtlModule.DllRegisterServer();
+	return hr;
+}
+
+
+// DllUnregisterServer - Removes entries from the system registry
+STDAPI DllUnregisterServer(void)
+{
+	HRESULT hr = _AtlModule.DllUnregisterServer();
+	return hr;
+}
+

tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.def

@@ -0,0 +1,9 @@
+; MeasurePageLoadTime.def : Declares the module parameters.
+
+LIBRARY      "MeasurePageLoadTime.DLL"
+
+EXPORTS
+	DllCanUnloadNow		PRIVATE
+	DllGetClassObject	PRIVATE
+	DllRegisterServer	PRIVATE
+	DllUnregisterServer	PRIVATE

tools/measure_page_load_time/ie_bho/MeasurePageLoadTime.idl

@@ -0,0 +1,65 @@
+// Copyright 2008, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// MeasurePageLoadTime.idl : IDL source for MeasurePageLoadTime
+//
+
+// This file will be processed by the MIDL tool to
+// produce the type library (MeasurePageLoadTime.tlb) and marshalling code.
+
+import "oaidl.idl";
+import "ocidl.idl";
+
+[
+	object,
+	uuid(019637EB-B865-485B-9A66-419477EE55A0),
+	dual,
+	nonextensible,
+	helpstring("IMeasurePageLoadTimeBHO Interface"),
+	pointer_default(unique)
+]
+interface IMeasurePageLoadTimeBHO : IDispatch{
+};
+[
+	uuid(61AC7AC4-B715-4955-A238-5F9AEA80DF4B),
+	version(1.0),
+	helpstring("MeasurePageLoadTime 1.0 Type Library")
+]
+library MeasurePageLoadTimeLib
+{
+	importlib("stdole2.tlb");
+	[
+		uuid(807E68BC-238F-4163-AE4B-0A3604F3E145),
+		helpstring("MeasurePageLoadTimeBHO Class")
+	]
+	coclass MeasurePageLoadTimeBHO
+	{
+		[default] interface IMeasurePageLoadTimeBHO;
+	};
+};