4 // Copyright (C) 2006 Ingo Ruhnke <grumbel@gmx.de>
6 // This program is free software; you can redistribute it and/or
7 // modify it under the terms of the GNU General Public License
8 // as published by the Free Software Foundation; either version 2
9 // of the License, or (at your option) any later version.
11 // This program is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // You should have received a copy of the GNU General Public License
17 // along with this program; if not, write to the Free Software
18 // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include <sys/types.h>
31 #include "tinygettext.hpp"
33 #include "physfs/physfs_stream.hpp"
34 #include <unison/vfs/FileSystem.hpp>
36 #include "findlocale.hpp"
38 //#define TRANSLATION_DEBUG
40 namespace TinyGetText {
42 /** Convert \a which is in \a from_charset to \a to_charset and return it */
43 std::string convert(const std::string& text,
44 const std::string& from_charset,
45 const std::string& to_charset)
47 if (from_charset == to_charset)
50 char *in = new char[text.length() + 1];
51 strcpy(in, text.c_str());
52 char *out = SDL_iconv_string(to_charset.c_str(), from_charset.c_str(), in, text.length() + 1);
56 log_warning << "Error: conversion from " << from_charset << " to " << to_charset << " failed" << std::endl;
63 iconv_t cd = SDL_iconv_open(to_charset.c_str(), from_charset.c_str());
65 size_t in_len = text.length();
66 size_t out_len = text.length()*3; // FIXME: cross fingers that this is enough
68 char* out_orig = new char[out_len];
69 char* in_orig = new char[in_len+1];
70 strcpy(in_orig, text.c_str());
73 ICONV_CONST char* in = in_orig;
74 size_t out_len_temp = out_len; // iconv is counting down the bytes it has
75 // written from this...
77 size_t retval = SDL_iconv(cd, &in, &in_len, &out, &out_len_temp);
78 out_len -= out_len_temp; // see above
79 if (retval == (size_t) -1)
81 log_warning << strerror(errno) << std::endl;
82 log_warning << "Error: conversion from " << from_charset << " to " << to_charset << " went wrong: " << retval << std::endl;
87 std::string ret(out_orig, out_len);
94 bool has_suffix(const std::string& lhs, const std::string rhs)
96 if (lhs.length() < rhs.length())
99 return lhs.compare(lhs.length() - rhs.length(), rhs.length(), rhs) == 0;
102 bool has_prefix(const std::string& lhs, const std::string rhs)
104 if (lhs.length() < rhs.length())
107 return lhs.compare(0, rhs.length(), rhs) == 0;
110 int plural1(int ) { return 0; }
111 int plural2_1(int n) { return (n != 1); }
112 int plural2_2(int n) { return (n > 1); }
113 int plural3_lv(int n) { return (n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2); }
114 int plural3_ga(int n) { return n==1 ? 0 : n==2 ? 1 : 2; }
115 int plural3_lt(int n) { return (n%10==1 && n%100!=11 ? 0 : n%10>=2 && (n%100<10 || n%100>=20) ? 1 : 2); }
116 int plural3_1(int n) { return (n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2); }
117 int plural3_sk(int n) { return (n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2; }
118 int plural3_pl(int n) { return (n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2); }
119 int plural3_sl(int n) { return (n%100==1 ? 0 : n%100==2 ? 1 : n%100==3 || n%100==4 ? 2 : 3); }
121 /** Language Definitions */
123 LanguageDef lang_hu("hu", "Hungarian", 1, plural1); // "nplurals=1; plural=0;"
124 LanguageDef lang_ja("ja", "Japanese", 1, plural1); // "nplurals=1; plural=0;"
125 LanguageDef lang_ko("ko", "Korean", 1, plural1); // "nplurals=1; plural=0;"
126 LanguageDef lang_tr("tr", "Turkish", 1, plural1); // "nplurals=1; plural=0;"
127 LanguageDef lang_da("da", "Danish", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
128 LanguageDef lang_nl("nl", "Dutch", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
129 LanguageDef lang_en("en", "English", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
130 LanguageDef lang_fo("fo", "Faroese", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
131 LanguageDef lang_de("de", "German", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
132 LanguageDef lang_nb("nb", "Norwegian Bokmal", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
133 LanguageDef lang_no("no", "Norwegian", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
134 LanguageDef lang_nn("nn", "Norwegian Nynorsk", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
135 LanguageDef lang_sv("sv", "Swedish", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
136 LanguageDef lang_et("et", "Estonian", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
137 LanguageDef lang_fi("fi", "Finnish", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
138 LanguageDef lang_el("el", "Greek", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
139 LanguageDef lang_he("he", "Hebrew", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
140 LanguageDef lang_it("it", "Italian", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
141 LanguageDef lang_pt("pt", "Portuguese", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
142 LanguageDef lang_es("es", "Spanish", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
143 LanguageDef lang_eo("eo", "Esperanto", 2, plural2_1); // "nplurals=2; plural=(n != 1);"
144 LanguageDef lang_fr("fr", "French", 2, plural2_2); // "nplurals=2; plural=(n > 1);"
145 LanguageDef lang_pt_BR("pt_BR", "Brazilian", 2, plural2_2); // "nplurals=2; plural=(n > 1);"
146 LanguageDef lang_lv("lv", "Latvian", 3, plural3_lv); // "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n != 0 ? 1 : 2);"
147 LanguageDef lang_ga("ga", "Irish", 3, plural3_ga); // "nplurals=3; plural=n==1 ? 0 : n==2 ? 1 : 2;"
148 LanguageDef lang_lt("lt", "Lithuanian", 3, plural3_lt); // "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && (n%100<10 || n%100>=20) ? 1 : 2);"
149 LanguageDef lang_hr("hr", "Croatian", 3, plural3_1); // "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);"
150 LanguageDef lang_cs("cs", "Czech", 3, plural3_1); // "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);"
151 LanguageDef lang_ru("ru", "Russian", 3, plural3_1); // "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);"
152 LanguageDef lang_uk("uk", "Ukrainian", 3, plural3_1); // "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);"
153 LanguageDef lang_sk("sk", "Slovak", 3, plural3_sk); // "nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;"
154 LanguageDef lang_pl("pl", "Polish", 3, plural3_pl); // "nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);
155 LanguageDef lang_sl("sl", "Slovenian", 3, plural3_sl); // "nplurals=4; plural=(n%100==1 ? 0 : n%100==2 ? 1 : n%100==3 || n%100==4 ? 2 : 3);"
159 get_language_def(const std::string& name)
161 if (name == "hu") return lang_hu;
162 else if (name == "ja") return lang_ja;
163 else if (name == "ko") return lang_ko;
164 else if (name == "tr") return lang_tr;
165 else if (name == "da") return lang_da;
166 else if (name == "nl") return lang_nl;
167 else if (name == "en") return lang_en;
168 else if (name == "fo") return lang_fo;
169 else if (name == "de") return lang_de;
170 else if (name == "nb") return lang_nb;
171 else if (name == "no") return lang_no;
172 else if (name == "nn") return lang_nn;
173 else if (name == "sv") return lang_sv;
174 else if (name == "et") return lang_et;
175 else if (name == "fi") return lang_fi;
176 else if (name == "el") return lang_el;
177 else if (name == "he") return lang_he;
178 else if (name == "it") return lang_it;
179 else if (name == "pt") return lang_pt;
180 else if (name == "es") return lang_es;
181 else if (name == "eo") return lang_eo;
182 else if (name == "fr") return lang_fr;
183 else if (name == "pt_BR") return lang_pt_BR;
184 else if (name == "lv") return lang_lv;
185 else if (name == "ga") return lang_ga;
186 else if (name == "lt") return lang_lt;
187 else if (name == "hr") return lang_hr;
188 else if (name == "cs") return lang_cs;
189 else if (name == "ru") return lang_ru;
190 else if (name == "uk") return lang_uk;
191 else if (name == "sk") return lang_sk;
192 else if (name == "pl") return lang_pl;
193 else if (name == "sl") return lang_sl;
197 DictionaryManager::DictionaryManager()
198 : current_dict(&empty_dict)
200 parseLocaleAliases();
201 // Environment variable SUPERTUX_LANG overrides language settings.
202 const char* lang = getenv( "SUPERTUX_LANG" );
204 set_language( lang );
207 // use findlocale to setup language
209 FL_FindLocale( &locale, FL_MESSAGES );
211 if (locale->country) {
212 set_language( std::string(locale->lang)+"_"+std::string(locale->country) );
214 set_language( std::string(locale->lang) );
217 FL_FreeLocale( &locale );
221 DictionaryManager::parseLocaleAliases()
223 // try to parse language alias list
224 std::ifstream in("/usr/share/locale/locale.alias");
227 while(in.good() && !in.eof()) {
228 while(isspace(c) && !in.eof())
231 if(c == '#') { // skip comments
232 while(c != '\n' && !in.eof())
238 while(!isspace(c) && !in.eof()) {
242 while(isspace(c) && !in.eof())
244 std::string language;
245 while(!isspace(c) && !in.eof()) {
252 set_language_alias(alias, language);
257 DictionaryManager::get_dictionary(const std::string& spec)
260 //log_debug << "Dictionary for language \"" << spec << "\" requested" << std::endl;
262 std::string lang = get_language_from_spec(spec);
264 //log_debug << "...normalized as \"" << lang << "\"" << std::endl;
266 Dictionaries::iterator i = dictionaries.find(get_language_from_spec(lang));
267 if (i != dictionaries.end())
271 else // Dictionary for languages lang isn't loaded, so we load it
273 //log_debug << "get_dictionary: " << lang << std::endl;
274 Dictionary& dict = dictionaries[lang];
276 dict.set_language(get_language_def(lang));
278 dict.set_charset(charset);
280 for (SearchPath::iterator p = search_path.begin(); p != search_path.end(); ++p)
282 std::vector<std::string> files = Unison::VFS::FileSystem::get().ls(*p);
283 for(std::vector<std::string>::iterator iter = files.begin();iter != files.end();++iter)
285 // check if filename matches requested language
286 std::string fname = *iter;
287 std::string load_from_file = "";
288 if(fname == lang + ".po") {
289 load_from_file = fname;
291 std::string::size_type s = lang.find("_");
292 if(s != std::string::npos) {
293 std::string lang_short = std::string(lang, 0, s);
294 if (fname == lang_short + ".po") {
295 load_from_file = lang_short;
300 // if it matched, load dictionary
301 if (load_from_file != "") {
302 //log_debug << "Loading dictionary for language \"" << lang << "\" from \"" << filename << "\"" << std::endl;
303 std::string pofile = *p + "/" + *iter;
305 IFileStream in(pofile);
306 read_po_file(dict, in);
307 } catch(std::exception& e) {
308 log_warning << "Error: Failure file opening: " << pofile << std::endl;
309 log_warning << e.what() << "" << std::endl;
315 char** files = PHYSFS_enumerateFiles(p->c_str());
318 log_warning << "Error: enumerateFiles() failed on " << *p << std::endl;
322 for(const char* const* filename = files;
323 *filename != 0; filename++) {
325 // check if filename matches requested language
326 std::string fname = std::string(*filename);
327 std::string load_from_file = "";
328 if(fname == lang + ".po") {
329 load_from_file = fname;
331 std::string::size_type s = lang.find("_");
332 if(s != std::string::npos) {
333 std::string lang_short = std::string(lang, 0, s);
334 if (fname == lang_short + ".po") {
335 load_from_file = lang_short;
340 // if it matched, load dictionary
341 if (load_from_file != "") {
342 //log_debug << "Loading dictionary for language \"" << lang << "\" from \"" << filename << "\"" << std::endl;
343 std::string pofile = *p + "/" + *filename;
345 IFileStream in(pofile);
346 read_po_file(dict, in);
347 } catch(std::exception& e) {
348 log_warning << "Error: Failure file opening: " << pofile << std::endl;
349 log_warning << e.what() << "" << std::endl;
354 PHYSFS_freeList(files);
363 std::set<std::string>
364 DictionaryManager::get_languages()
366 std::set<std::string> languages;
368 for (SearchPath::iterator p = search_path.begin(); p != search_path.end(); ++p)
370 std::vector<std::string> files = Unison::VFS::FileSystem::get().ls(*p);
371 for(std::vector<std::string>::iterator iter = files.begin();iter != files.end();++iter)
373 if(has_suffix(*iter, ".po")) {
374 std::string filename = *iter;
375 languages.insert(filename.substr(0, filename.length()-3));
379 char** files = PHYSFS_enumerateFiles(p->c_str());
382 log_warning << "Error: opendir() failed on " << *p << std::endl;
386 for(const char* const* file = files; *file != 0; file++) {
387 if(has_suffix(*file, ".po")) {
388 std::string filename = *file;
389 languages.insert(filename.substr(0, filename.length()-3));
392 PHYSFS_freeList(files);
400 DictionaryManager::set_language(const std::string& lang)
402 //log_debug << "set_language \"" << lang << "\"" << std::endl;
403 language = get_language_from_spec(lang);
404 //log_debug << "==> \"" << language << "\"" << std::endl;
405 current_dict = & (get_dictionary(language));
409 DictionaryManager::get_language() const
415 DictionaryManager::set_charset(const std::string& charset)
417 dictionaries.clear(); // changing charset invalidates cache
418 this->charset = charset;
419 set_language(language);
423 DictionaryManager::set_language_alias(const std::string& alias,
424 const std::string& language)
426 language_aliases.insert(std::make_pair(alias, language));
430 DictionaryManager::get_language_from_spec(const std::string& spec)
432 std::string lang = spec;
433 Aliases::iterator i = language_aliases.find(lang);
434 if(i != language_aliases.end()) {
438 std::string::size_type s = lang.find(".");
439 if(s != std::string::npos) {
440 lang = std::string(lang, 0, s);
444 if(s == std::string::npos) {
445 std::string lang_big = lang;
446 std::transform (lang_big.begin(), lang_big.end(), lang_big.begin(), toupper);
447 lang += "_" + lang_big;
455 DictionaryManager::add_directory(const std::string& pathname)
457 dictionaries.clear(); // adding directories invalidates cache
458 search_path.push_back(pathname);
459 set_language(language);
462 //---------------------------------------------------------------------------
464 Dictionary::Dictionary(const LanguageDef& language_, const std::string& charset_)
465 : language(language_), charset(charset_)
469 Dictionary::Dictionary()
475 Dictionary::get_charset() const
481 Dictionary::set_charset(const std::string& charset_)
487 Dictionary::set_language(const LanguageDef& lang)
493 Dictionary::translate(const std::string& msgid, const std::string& msgid2, int num)
495 PluralEntries::iterator i = plural_entries.find(msgid);
496 std::map<int, std::string>& msgstrs = i->second;
498 if (i != plural_entries.end() && !msgstrs.empty())
500 int g = language.plural(num);
501 std::map<int, std::string>::iterator j = msgstrs.find(g);
502 if (j != msgstrs.end())
508 // Return the first translation, in case we can't translate the specific number
509 return msgstrs.begin()->second;
514 #ifdef TRANSLATION_DEBUG
515 log_warning << "Couldn't translate: " << msgid << std::endl;
516 log_warning << "Candidates: " << std::endl;
517 for (PluralEntries::iterator i = plural_entries.begin(); i != plural_entries.end(); ++i)
518 log_debug << "'" << i->first << "'" << std::endl;
521 if (plural2_1(num)) // default to english rules
529 Dictionary::translate(const char* msgid)
531 Entries::iterator i = entries.find(msgid);
532 if (i != entries.end() && !i->second.empty())
534 return i->second.c_str();
538 #ifdef TRANSLATION_DBEUG
539 log_warning << "Couldn't translate: " << msgid << std::endl;
546 Dictionary::translate(const std::string& msgid)
548 Entries::iterator i = entries.find(msgid);
549 if (i != entries.end() && !i->second.empty())
555 #ifdef TRANSLATION_DBEUG
556 log_warning << "Couldn't translate: " << msgid << std::endl;
563 Dictionary::add_translation(const std::string& msgid, const std::string& ,
564 const std::map<int, std::string>& msgstrs)
566 // Do we need msgid2 for anything? its after all supplied to the
567 // translate call, so we just throw it away
568 plural_entries[msgid] = msgstrs;
572 Dictionary::add_translation(const std::string& msgid, const std::string& msgstr)
574 entries[msgid] = msgstr;
588 std::string from_charset;
589 std::string to_charset;
591 std::string current_msgid;
592 std::string current_msgid_plural;
593 std::map<int, std::string> msgstr_plural;
597 enum { WANT_MSGID, WANT_MSGSTR, WANT_MSGSTR_PLURAL, WANT_MSGID_PLURAL } state;
600 POFileReader(std::istream& in, Dictionary& dict_)
606 if(c == (char) 0xef) { // skip UTF-8 intro that some texteditors produce
615 void parse_header(const std::string& header)
617 // Seperate the header in lines
618 typedef std::vector<std::string> Lines;
621 std::string::size_type start = 0;
622 for(std::string::size_type i = 0; i < header.length(); ++i)
624 if (header[i] == '\n')
626 lines.push_back(header.substr(start, i - start));
631 for(Lines::iterator i = lines.begin(); i != lines.end(); ++i)
633 if (has_prefix(*i, "Content-Type: text/plain; charset=")) {
634 from_charset = i->substr(strlen("Content-Type: text/plain; charset="));
638 if (from_charset.empty() || from_charset == "CHARSET")
640 log_warning << "Error: Charset not specified for .po, fallback to ISO-8859-1" << std::endl;
641 from_charset = "ISO-8859-1";
644 to_charset = dict.get_charset();
645 if (to_charset.empty())
646 { // No charset requested from the dict, use utf-8
647 to_charset = "utf-8";
648 dict.set_charset(from_charset);
652 void add_token(const Token& token)
657 if (token.keyword == "msgid")
659 current_msgid = token.content;
660 state = WANT_MSGID_PLURAL;
662 else if (token.keyword.empty())
664 //log_warning << "Got EOF, everything looks ok." << std::endl;
668 log_warning << "tinygettext: expected 'msgid' keyword, got " << token.keyword << " at line " << line_num << std::endl;
672 case WANT_MSGID_PLURAL:
673 if (token.keyword == "msgid_plural")
675 current_msgid_plural = token.content;
676 state = WANT_MSGSTR_PLURAL;
686 if (token.keyword == "msgstr")
688 if (current_msgid == "")
689 { // .po Header is hidden in the msgid with the empty string
690 parse_header(token.content);
694 dict.add_translation(current_msgid, convert(token.content, from_charset, to_charset));
700 log_warning << "tinygettext: expected 'msgstr' keyword, got " << token.keyword << " at line " << line_num << std::endl;
704 case WANT_MSGSTR_PLURAL:
705 if (has_prefix(token.keyword, "msgstr["))
708 if (sscanf(token.keyword.c_str(), "msgstr[%d]", &num) != 1)
710 log_warning << "Error: Couldn't parse: " << token.keyword << std::endl;
714 msgstr_plural[num] = convert(token.content, from_charset, to_charset);
719 dict.add_translation(current_msgid, current_msgid_plural, msgstr_plural);
728 inline int getchar(std::istream& in)
736 void tokenize_po(std::istream& in)
738 enum State { READ_KEYWORD,
740 READ_CONTENT_IN_STRING,
743 State state = READ_KEYWORD;
747 while((c = getchar(in)) != EOF)
749 //log_debug << "Lexing char: " << char(c) << " " << state << std::endl;
755 state = SKIP_COMMENT;
764 } while((c = getchar(in)) != EOF && !isspace(c));
767 state = READ_CONTENT;
772 while((c = getchar(in)) != EOF)
775 // Found start of content
776 state = READ_CONTENT_IN_STRING;
778 } else if (isspace(c)) {
780 } else { // Read something that may be a keyword
782 state = READ_KEYWORD;
789 case READ_CONTENT_IN_STRING:
794 if (c == 'n') token.content += '\n';
795 else if (c == 't') token.content += '\t';
796 else if (c == 'r') token.content += '\r';
797 else if (c == '"') token.content += '"';
798 else if (c == '\\') token.content += '\\';
801 log_warning << "Unhandled escape character: " << char(c) << std::endl;
806 log_warning << "Unterminated string" << std::endl;
808 } else if (c == '"') { // Content string is terminated
809 state = READ_CONTENT;
817 state = READ_KEYWORD;
825 void read_po_file(Dictionary& dict_, std::istream& in)
827 POFileReader reader(in, dict_);
830 } // namespace TinyGetText