From e40edade0e005a28f7db0a7e6dc51a2acb734aae Mon Sep 17 00:00:00 2001 From: Andrzej Rybczak Date: Thu, 4 Oct 2012 21:25:48 +0200 Subject: [PATCH] use boost.locale for charset conversions instead of iconv --- configure.in | 47 +++++--------- src/charset.cpp | 120 ++++-------------------------------- src/charset.h | 24 +------- src/cmdargs.cpp | 5 +- src/lastfm.cpp | 8 +-- src/lyrics.cpp | 8 +-- src/lyrics_fetcher.cpp | 4 +- src/media_library.cpp | 1 + src/ncmpcpp.cpp | 4 +- src/status.cpp | 6 +- src/utility/wide_string.cpp | 24 ++------ src/utility/wide_string.h | 4 +- 12 files changed, 53 insertions(+), 202 deletions(-) diff --git a/configure.in b/configure.in index e0fc623b..3bc3c8bb 100644 --- a/configure.in +++ b/configure.in @@ -15,7 +15,6 @@ AC_ARG_ENABLE(clock, AS_HELP_STRING([--enable-clock], [Enable clock screen @<:@d AC_ARG_ENABLE(unicode, AS_HELP_STRING([--enable-unicode], [Enable utf8 support @<:@default=yes@:>@]), [unicode=$enableval], [unicode=yes]) AC_ARG_WITH(curl, AS_HELP_STRING([--with-curl], [Enable fetching lyrics from the Internet @<:@default=auto@:>@]), [curl=$withval], [curl=auto]) AC_ARG_WITH(fftw, AS_HELP_STRING([--with-fftw], [Enable fftw support (required for frequency spectrum vizualization) @<:@default=auto@:>@]), [fftw=$withval], [fftw=auto]) -AC_ARG_WITH(iconv, AS_HELP_STRING([--with-iconv], [Enable iconv support (Note: if you use utf-8 system wide, you can disable this) @<:@default=auto@:>@]), [iconv=$withval], [iconv=auto]) AC_ARG_WITH(pdcurses, AS_HELP_STRING([--with-pdcurses[=LIBNAME]], [Link against pdcurses instead of ncurses @<:@default=XCurses@:>@]), [pdcurses=$withval], [pdcurses=no]) AC_ARG_WITH(taglib, AS_HELP_STRING([--with-taglib], [Enable tag editor @<:@default=auto@:>@]), [taglib=$withval], [taglib=auto]) @@ -27,19 +26,6 @@ if test "$clock" = "yes"; then AC_DEFINE([ENABLE_CLOCK], [1], [enables clock screen]) fi -dnl ===================================== -dnl = checking for -fno-exceptions flag = -dnl ===================================== -AC_MSG_CHECKING([whether compiler supports -fno-exceptions]) -old_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="-fno-exceptions" -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]])], - AC_MSG_RESULT([yes]) - no_exceptions="-fno-exceptions", - AC_MSG_RESULT([no]) -) -CXXFLAGS="$old_CXXFLAGS $no_exceptions" - dnl ================================ dnl = checking for -std=c++0x flag = dnl ================================ @@ -96,6 +82,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[struct A { virtual void foo() { } }; struct AC_DEFINE([OVERRIDE], []), ) +dnl ============================= +dnl = setting boost environment = +dnl ============================= +AS_IF([test -z "${BOOST_LIB_SUFFIX+x}"], [BOOST_LIB_SUFFIX=-mt]) +AC_ARG_VAR([BOOST_LIB_SUFFIX], [Boost library name suffix [default=-mt]]) + +dnl ============================= +dnl = checking for boost.locale = +dnl ============================= +AC_CHECK_HEADERS([boost/locale/encoding.hpp], , + AC_MSG_ERROR(boost/locale/encoding.hpp is missing) +) +AC_CHECK_LIB(boost_locale$BOOST_LIB_SUFFIX, main, LDFLAGS="$LDFLAGS -lboost_locale$BOOST_LIB_SUFFIX", + AC_MSG_ERROR([no boost.locale library found]) +) + dnl ============================== dnl = checking for regex (win32) = dnl ============================== @@ -132,23 +134,6 @@ AC_CHECK_HEADERS([pthread.h], ), ) -dnl ====================== -dnl = checking for iconv = -dnl ====================== -if test "$iconv" != "no" ; then - AC_CHECK_HEADERS([iconv.h], - AC_MSG_CHECKING([whether iconv takes const char **]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[iconv(0, (const char **)0, 0, 0, 0);]])], - AC_MSG_RESULT([yes]) AC_DEFINE([ICONV_CONST], [const], [pass const pointer to iconv]), - AC_MSG_RESULT([no]) AC_DEFINE([ICONV_CONST], [], [pass non-const pointer to iconv])) - AC_CHECK_LIB(iconv, libiconv, LDFLAGS="$LDFLAGS -liconv", ) - , - if test "$iconv" = "yes"; then - AC_MSG_ERROR([iconv.h header is required]) - fi - ) -fi - dnl ======================== dnl = checking for ncurses = dnl ======================== diff --git a/src/charset.cpp b/src/charset.cpp index 0941419d..2d0f8569 100644 --- a/src/charset.cpp +++ b/src/charset.cpp @@ -18,130 +18,34 @@ * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ +#include #include "charset.h" - -#ifdef HAVE_ICONV_H - -#include -#include -#include -#include -#include -#include - #include "settings.h" -namespace {// +namespace Charset {// -bool is_utf8(const char *s) +std::string toUtf8From(std::string s, const char *charset) { - for (; *s; ++s) - { - if (*s & 0x80) // 1xxxxxxx - { - char c = 0x40; - unsigned i = 0; - while (c & *s) - ++i, c >>= 1; - if (i < 1 || i > 3) // not 110xxxxx, 1110xxxx, 11110xxx - return false; - for (unsigned j = 0; j < i; ++j) - if (!*++s || !(*s & 0x80) || *s & 0x40) // 10xxxxxx - return false; - } - } - return true; + return boost::locale::conv::to_utf(s, charset); } -bool has_non_ascii_chars(const char *s) +std::string fromUtf8To(std::string s, const char *charset) { - for (; *s; ++s) - if (*s & 0x80) - return true; - return false; -} - -void charset_convert(const char *from, const char *to, const char *&inbuf, - bool delete_old, size_t len = 0) -{ - assert(inbuf); - assert(from); - assert(to); - - iconv_t cd = iconv_open(to, from); - if (cd == iconv_t(-1)) - { - std::cerr << "Error while executing iconv_open: " << strerror(errno) << "\n"; - return; - } - - if (!len) - len = strlen(inbuf); - size_t buflen = len*MB_CUR_MAX+1; - char *outbuf = static_cast(malloc(buflen)); - char *outstart = outbuf; - const char *instart = inbuf; - - if (iconv(cd, const_cast(&inbuf), &len, &outbuf, &buflen) == size_t(-1)) - { - std::cerr << "Error while executing iconv: " << strerror(errno) << "\n"; - inbuf = instart; - delete [] outstart; - } - else - { - *outbuf = 0; - if (delete_old) - free(const_cast(instart)); - inbuf = outstart; - } - iconv_close(cd); -} - -} - -namespace IConv {// - -void convertFromTo(const char *from, const char *to, std::string &s) -{ - const char *tmp = strdup(s.c_str()); - charset_convert(from, to, tmp, true, s.length()); - s = tmp; - free(const_cast(tmp)); + return boost::locale::conv::to_utf(s, charset); } std::string utf8ToLocale(std::string s) { - utf8ToLocale_(s); - return s; -} - -void utf8ToLocale_(std::string &s) -{ - if (Config.system_encoding.empty() || !has_non_ascii_chars(s.c_str())) - return; - const char *tmp = strdup(s.c_str()); - charset_convert("utf-8", Config.system_encoding.c_str(), tmp, 1, s.length()); - s = tmp; - free(const_cast(tmp)); + return Config.system_encoding.empty() + ? s + : boost::locale::conv::from_utf(s, Config.system_encoding); } std::string localeToUtf8(std::string s) { - localeToUtf8_(s); - return s; -} - -void localeToUtf8_(std::string &s) -{ - if (Config.system_encoding.empty() || !has_non_ascii_chars(s.c_str()) || is_utf8(s.c_str())) - return; - const char *tmp = strdup(s.c_str()); - charset_convert(Config.system_encoding.c_str(), "utf-8", tmp, 1, s.length()); - s = tmp; - free(const_cast(tmp)); + return Config.system_encoding.empty() + ? s + : boost::locale::conv::to_utf(s, Config.system_encoding); } } - -#endif // HAVE_ICONV_H diff --git a/src/charset.h b/src/charset.h index d735425e..99188dc5 100644 --- a/src/charset.h +++ b/src/charset.h @@ -21,34 +21,16 @@ #ifndef NCMPCPP_CHARSET_H #define NCMPCPP_CHARSET_H -#include "config.h" #include -namespace IConv {// +namespace Charset {// -#ifdef HAVE_ICONV_H - -void convertFromTo(const char *from, const char *to, std::string &s); +std::string toUtf8From(std::string s, const char *charset); +std::string fromUtf8To(std::string s, const char *charset); std::string utf8ToLocale(std::string s); std::string localeToUtf8(std::string s); -void utf8ToLocale_(std::string &s); -void localeToUtf8_(std::string &s); - -#else - -inline void convertFromTo(const char *, const char *, std::string &) { } - -inline std::string utf8ToLocale(std::string s) { return s; } -inline std::string localeToUtf8(std::string s) { return s; } - -inline void utf8ToLocale_(std::string &) { } -inline void localeToUtf8_(std::string &) { } - -#endif // HAVE_ICONV_H - } #endif // NCMPCPP_CHARSET_H - diff --git a/src/cmdargs.cpp b/src/cmdargs.cpp index 9367ff12..6222009b 100644 --- a/src/cmdargs.cpp +++ b/src/cmdargs.cpp @@ -89,9 +89,6 @@ void ParseArgv(int argc, char **argv) # ifdef HAVE_CURL_CURL_H << " curl" # endif -# ifdef HAVE_ICONV_H - << " iconv" -# endif # ifdef HAVE_FFTW3_H << " fftw" # endif @@ -188,7 +185,7 @@ void ParseArgv(int argc, char **argv) replace(now_playing_format, "\\t", "\t"); } } - std::cout << IConv::utf8ToLocale( + std::cout << Charset::utf8ToLocale( Mpd.GetCurrentlyPlayingSong().toString(now_playing_format, Config.tags_separator)) << "\n"; } exit(0); diff --git a/src/lastfm.cpp b/src/lastfm.cpp index 4d58dfe6..32183d37 100644 --- a/src/lastfm.cpp +++ b/src/lastfm.cpp @@ -127,12 +127,11 @@ void Lastfm::Load() { bool first = 1; std::string line; - while (getline(input, line)) + while (std::getline(input, line)) { if (!first) w << '\n'; - IConv::utf8ToLocale_(line); - w << line; + w << Charset::utf8ToLocale(line); first = 0; } input.close(); @@ -171,8 +170,7 @@ void Lastfm::Download() { Save(result.second); w.clear(); - IConv::utf8ToLocale_(result.second); - w << result.second; + w << Charset::utf8ToLocale(result.second); itsService->colorizeOutput(w); } else diff --git a/src/lyrics.cpp b/src/lyrics.cpp index bf1053a1..42bd7b4a 100644 --- a/src/lyrics.cpp +++ b/src/lyrics.cpp @@ -249,8 +249,7 @@ void *Lyrics::Download() { Save(itsFilename, result.second); w.clear(); - IConv::utf8ToLocale_(result.second); - w << result.second; + w << Charset::utf8ToLocale(result.second); } else w << '\n' << L"Lyrics weren't found."; @@ -320,12 +319,11 @@ void Lyrics::Load() { bool first = 1; std::string line; - while (getline(input, line)) + while (std::getline(input, line)) { if (!first) w << '\n'; - IConv::utf8ToLocale_(line); - w << line; + w << Charset::utf8ToLocale(line); first = 0; } w.flush(); diff --git a/src/lyrics_fetcher.cpp b/src/lyrics_fetcher.cpp index 998197a1..94c8a19e 100644 --- a/src/lyrics_fetcher.cpp +++ b/src/lyrics_fetcher.cpp @@ -212,7 +212,7 @@ void LyricstimeFetcher::postProcess(std::string &data) { // lyricstime.com uses iso-8859-1 as the encoding // so we need to convert obtained lyrics to utf-8 - IConv::convertFromTo("iso-8859-1", "utf-8", data); + data = Charset::toUtf8From(data, "iso-8859-1"); LyricsFetcher::postProcess(data); } @@ -245,7 +245,7 @@ void LyricsmaniaFetcher::postProcess(std::string &data) { // lyricsmania.com uses iso-8859-1 as the encoding // so we need to convert obtained lyrics to utf-8 - IConv::convertFromTo("iso-8859-1", "utf-8", data); + data = Charset::toUtf8From(data, "iso-8859-1"); LyricsFetcher::postProcess(data); } diff --git a/src/media_library.cpp b/src/media_library.cpp index 5ed32127..afddae46 100644 --- a/src/media_library.cpp +++ b/src/media_library.cpp @@ -314,6 +314,7 @@ void MediaLibrary::update() } if (idx < Tags.size()) Tags.resizeList(idx); + std::sort(Tags.beginV(), Tags.endV(), SortPrimaryTags()); }); Tags.refresh(); } diff --git a/src/ncmpcpp.cpp b/src/ncmpcpp.cpp index eb43df69..f09b65c5 100644 --- a/src/ncmpcpp.cpp +++ b/src/ncmpcpp.cpp @@ -89,8 +89,8 @@ int main(int argc, char **argv) using Global::VolumeState; using Global::Timer; - srand(time(0)); - setlocale(LC_ALL, ""); + std::srand(std::time(0)); + std::setlocale(LC_ALL, ""); std::locale::global(std::locale("")); Config.CheckForCommandLineConfigFilePath(argv, argc); diff --git a/src/status.cpp b/src/status.cpp index 2420f0e4..edf22857 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -330,8 +330,8 @@ void Status::Changes::elapsedTime() } NC::WBuffer first, second; - stringToBuffer(ToWString(IConv::utf8ToLocale(np.toString(Config.new_header_first_line, Config.tags_separator, "$"))), first); - stringToBuffer(ToWString(IConv::utf8ToLocale(np.toString(Config.new_header_second_line, Config.tags_separator, "$"))), second); + stringToBuffer(ToWString(Charset::utf8ToLocale(np.toString(Config.new_header_first_line, Config.tags_separator, "$"))), first); + stringToBuffer(ToWString(Charset::utf8ToLocale(np.toString(Config.new_header_second_line, Config.tags_separator, "$"))), second); size_t first_len = wideLength(first.str()); size_t first_margin = (std::max(tracklength.length()+1, VolumeState.length()))*2; @@ -382,7 +382,7 @@ void Status::Changes::elapsedTime() tracklength += "]"; } NC::WBuffer np_song; - stringToBuffer(ToWString(IConv::utf8ToLocale(np.toString(Config.song_status_format, Config.tags_separator, "$"))), np_song); + stringToBuffer(ToWString(Charset::utf8ToLocale(np.toString(Config.song_status_format, Config.tags_separator, "$"))), np_song); *wFooter << NC::XY(0, 1) << wclrtoeol << NC::fmtBold << player_state << NC::fmtBoldEnd; np_song.write(*wFooter, playing_song_scroll_begin, wFooter->getWidth()-player_state.length()-tracklength.length(), L" ** "); *wFooter << NC::fmtBold << NC::XY(wFooter->getWidth()-tracklength.length(), 1) << tracklength << NC::fmtBoldEnd; diff --git a/src/utility/wide_string.cpp b/src/utility/wide_string.cpp index 71c2383a..193c9cf2 100644 --- a/src/utility/wide_string.cpp +++ b/src/utility/wide_string.cpp @@ -18,31 +18,17 @@ * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ +#include #include "utility/wide_string.h" -std::string ToString(const std::wstring &ws) +std::string ToString(std::wstring ws) { - std::string result; - char s[MB_CUR_MAX]; - for (size_t i = 0; i < ws.length(); ++i) - { - int n = wcrtomb(s, ws[i], 0); - if (n > 0) - result.append(s, n); - } - return result; + return boost::locale::conv::utf_to_utf(ws); } -std::wstring ToWString(const std::string &s) +std::wstring ToWString(std::string s) { - std::wstring result; - wchar_t *ws = new wchar_t[s.length()]; - const char *c_s = s.c_str(); - int n = mbsrtowcs(ws, &c_s, s.length(), 0); - if (n > 0) - result.append(ws, n); - delete [] ws; - return result; + return boost::locale::conv::utf_to_utf(s); } size_t wideLength(const std::wstring &ws) diff --git a/src/utility/wide_string.h b/src/utility/wide_string.h index d3b3c9fa..58a013dd 100644 --- a/src/utility/wide_string.h +++ b/src/utility/wide_string.h @@ -23,8 +23,8 @@ #include -std::string ToString(const std::wstring &ws); -std::wstring ToWString(const std::string &s); +std::string ToString(std::wstring ws); +std::wstring ToWString(std::string s); size_t wideLength(const std::wstring &ws);