/*
 * mptString.cpp
 * -------------
 * Purpose: Small string-related utilities, number and message formatting.
 * Notes  : Currently none.
 * Authors: OpenMPT Devs
 * The OpenMPT source code is released under the BSD license. Read LICENSE for more details.
 */

#include "stdafx.h"
#include "mptString.h"

#include "Endianness.h"

#include <locale>
#include <string>
#include <stdexcept>
#include <vector>

#include <cstdlib>

#if defined(MODPLUG_TRACKER)
#include <cwctype>
#endif // MODPLUG_TRACKER

#if defined(MODPLUG_TRACKER)
#include <wctype.h>
#endif // MODPLUG_TRACKER

#if MPT_OS_WINDOWS
#include <windows.h>
#endif


OPENMPT_NAMESPACE_BEGIN



/*



Quick guide to the OpenMPT string type jungle
=============================================



This quick guide is only meant as a hint. There may be valid reasons to not
honor the recommendations found here. Staying consistent with surrounding and/or
related code sections may also be important.



List of string types
--------------------

 *  std::string (OpenMPT, libopenmpt)
    C++ string of unspecifed 8bit encoding. Try to always document the
    encoding if not clear from context. Do not use unless there is an obvious
    reason to do so.

 *  std::wstring (OpenMPT)
    UTF16 (on windows) or UTF32 (otherwise). Do not use unless there is an
    obvious reason to do so.

 *  mpt::lstring (OpenMPT)
    OpenMPT locale string type. The encoding is always CP_ACP. Do not use unless
    there is an obvious reason to do so.

 *  char* (OpenMPT, libopenmpt)
    C string of unspecified encoding. Use only for static literals or in
    performance critical inner loops where full control and avoidance of memory
    allocations is required.

 *  wchar_t* (OpenMPT)
    C wide string. Use only if Unicode is required for static literals or in
    performance critical inner loops where full control and avoidance of memory
    allocation is required.

 *  mpt::winstring (OpenMPT)
    OpenMPT type-safe string to interface with native WinAPI, either encoded in
    locale/CP_ACP (if !UNICODE) or UTF16 (if UNICODE).

 *  CString (OpenMPT)
    MFC string type, either encoded in locale/CP_ACP (if !UNICODE) or UTF16 (if
    UNICODE). Specify literals with _T(""). Use in MFC GUI code.

 *  CStringA (OpenMPT)
    MFC ANSI string type. The encoding is always CP_ACP. Do not use.

 *  CStringW (OpenMPT)
    MFC Unicode string type. Do not use.

 *  mpt::PathString (OpenMPT, libopenmpt)
    String type representing paths and filenames. Always use for these in order
    to avoid potentially lossy conversions. Use P_("") macro for
    literals.

 *  mpt::ustring (OpenMPT, libopenmpt)
    The default unicode string type. Can be encoded in UTF8 or UTF16 or UTF32,
    depending on MPT_USTRING_MODE_* and sizeof(wchar_t). Literals can written as
    U_(""). Use as your default string type if no other string type is
    a measurably better fit.

 *  MPT_UTF8 (OpenMPT, libopenmpt)
    Macro that generates a mpt::ustring from string literals containing
    non-ascii characters. In order to keep the source code in ascii encoding,
    always express non-ascii characters using explicit \x23 escaping. Note that
    depending on the underlying type of mpt::ustring, MPT_UTF8 *requires* a
    runtime conversion. Only use for string literals containing non-ascii
    characters (use MPT_USTRING otherwise).

 *  MPT_ULITERAL / MPT_UCHAR / mpt::uchar (OpenMPT, libopenmpt)
    Macros which generate string literals, char literals and the char literal
    type respectively. These are especially useful in constexpr contexts or
    global data where MPT_USTRING is either unusable or requires a global
    contructor to run. Do NOT use as a performance optimization in place of
    MPT_USTRING however, because MPT_USTRING can be converted to C++11/14 user
    defined literals eventually, while MPT_ULITERAL cannot because of constexpr
    requirements.

 *  mpt::RawPathString (OpenMPT, libopenmpt)
    Internal representation of mpt::PathString. Only use for parsing path
    fragments.

 *  mpt::u8string (OpenMPT, libopenmpt)
    Internal representation of mpt::ustring. Do not use directly. Ever.

 *  std::basic_string<char> (OpenMPT)
    Same as std::string. Do not use std::basic_string in the templated form.

 *  std::basic_string<wchar_t> (OpenMPT)
    Same as std::wstring. Do not use std::basic_string in the templated form.

The following string types are available in order to avoid the need to overload
functions on a huge variety of string types. Use only ever as function argument
types.
Note that the locale charset is not available on all libopenmpt builds (in which
case the option is ignored or a sensible fallback is used; these types are
always available).
All these types publicly inherit from mpt::ustring and do not contain any
additional state. This means that they work the same way as mpt::ustring does
and do support type-slicing for both, read and write accesses.
These types only add conversion constructors for all string types that have a
defined encoding and for all 8bit string types using the specified encoding
heuristic.

 *  AnyUnicodeString (OpenMPT, libopenmpt)
    Is constructible from any Unicode string.

 *  AnyString (OpenMPT, libopenmpt)
    Tries to do the smartest auto-magic we can do.

 *  AnyStringLocale (OpenMPT, libopenmpt)
    char-based strings are assumed to be in locale encoding.

 *  AnyStringUTF8orLocale (OpenMPT, libopenmpt)
    char-based strings are tried in UTF8 first, if this fails, locale is used.

 *  AnyStringUTF8 (OpenMPT, libopenmpt)
    char-based strings are assumed to be in UTF8.



Encoding of 8bit strings
------------------------

8bit strings have an unspecified encoding. When the string is contained within a
CSoundFile object, the encoding is most likely CSoundFile::GetCharsetInternal(),
otherwise, try to gather the most probable encoding from surrounding or related
code sections.



Decision tree to help deciding which string type to use
-------------------------------------------------------

if in libopenmpt
 if in libopenmpt c++ interface
  T = std::string, the encoding is utf8
 elif in libopenmpt c interface
  T = char*, the encoding is utf8
 elif performance critical inner loop
  T = char*, document the encoding if not clear from context 
 elif string literal containing non-ascii characters
  T = MPT_UTF8
 elif path or file
  if parsing path fragments
   T = mpt::RawPathString
       template your function on the concrete underlying string type
       (std::string and std::wstring) or use preprocessor MPT_OS_WINDOWS
  else
   T = mpt::PathString
  fi
 else
  T = mpt::ustring
 fi
else
 if performance critical inner loop
  if needs unicode support
   T = mpt::uchar* / MPT_ULITERAL
  else
   T = char*, document the encoding if not clear from context 
  fi
 elif string literal containing non-ascii characters
  T = MPT_UTF8
 elif path or file
  if parsing path fragments
   T = mpt::RawPathString
       template your function on the concrete underlying string type
       (std::string and std::wstring) or use preprocessor MPT_OS_WINDOWS
  else
   T = mpt::PathString
  fi
 elif winapi interfacing code
  T = mpt::winstring
 elif mfc/gui code
  T = CString
 else
  if constexpr context or global data
   T = mpt::uchar* / MPT_ULITERAL
  else
   T = mpt::ustring
  fi
 fi
fi

This boils down to: Prefer mpt::PathString and mpt::ustring, and only use any
other string type if there is an obvious reason to do so.



Character set conversions
-------------------------

Character set conversions in OpenMPT are always fuzzy.

Behaviour in case of an invalid source encoding and behaviour in case of an
unrepresentable destination encoding can be any of the following:
 *  The character is replaced by some replacement character ('?' or L'\ufffd' in
    most cases).
 *  The character is replaced by a similar character (either semantically
    similiar or visually similar).
 *  The character is transcribed with some ASCII text.
 *  The character is discarded.
 *  Conversion stops at this very character.

Additionally. conversion may stop or continue on \0 characters in the middle of
the string.

Behaviour can vary from one conversion tuple to any other.

If you need to ensure lossless conversion, do a roundtrip conversion and check
for equality.



Unicode handling
----------------

OpenMPT is generally not aware of and does not handle different Unicode
normalization forms.
You should be aware of the following possibilities:
 *  Conversion between UTF8, UTF16, UTF32 may or may not change between NFC and
    NFD.
 *  Conversion from any non-Unicode 8bit encoding can result in both, NFC or NFD
    forms.
 *  Conversion to any non-Unicode 8bit encoding may or may not involve
    conversion to NFC, NFD, NFKC or NFKD during the conversion. This in
    particular means that conversion of decomposed german umlauts to ISO8859-1
    may fail.
 *  Changing the normalization form of path strings may render the file
    inaccessible.

Unicode BOM may or may not be preserved and/or discarded during conversion.

Invalid Unicode code points may be treated as invalid or as valid characters
when converting between different Unicode encodings.



Interfacing with WinAPI
-----------------------

When in MFC code, use CString.
When in non MFC code, either use std::wstring when directly interfacing with
APIs only available in WCHAR variants, or use mpt::winstring and
mpt::WinStringBuf helpers otherwise.
Specify TCHAR string literals with _T("foo") in mptrack/, and with TEXT("foo")
in common/ or sounddev/. _T() requires <tchar.h> which is specific to the MSVC
runtime and not portable across compilers. TEXT() is from <windows.h>. We use
_T() in mptrack/ only because it is shorter.



*/



namespace mpt { namespace String {



/*
default 1:1 mapping
static constexpr char32_t CharsetTableISO8859_1[256] = {
	0x0000,0x0001,0x0002,0x0003,0x0004,0x0005,0x0006,0x0007,0x0008,0x0009,0x000a,0x000b,0x000c,0x000d,0x000e,0x000f,
	0x0010,0x0011,0x0012,0x0013,0x0014,0x0015,0x0016,0x0017,0x0018,0x0019,0x001a,0x001b,0x001c,0x001d,0x001e,0x001f,
	0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
	0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
	0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
	0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
	0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
	0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x007f,
	0x0080,0x0081,0x0082,0x0083,0x0084,0x0085,0x0086,0x0087,0x0088,0x0089,0x008a,0x008b,0x008c,0x008d,0x008e,0x008f,
	0x0090,0x0091,0x0092,0x0093,0x0094,0x0095,0x0096,0x0097,0x0098,0x0099,0x009a,0x009b,0x009c,0x009d,0x009e,0x009f,
	0x00a0,0x00a1,0x00a2,0x00a3,0x00a4,0x00a5,0x00a6,0x00a7,0x00a8,0x00a9,0x00aa,0x00ab,0x00ac,0x00ad,0x00ae,0x00af,
	0x00b0,0x00b1,0x00b2,0x00b3,0x00b4,0x00b5,0x00b6,0x00b7,0x00b8,0x00b9,0x00ba,0x00bb,0x00bc,0x00bd,0x00be,0x00bf,
	0x00c0,0x00c1,0x00c2,0x00c3,0x00c4,0x00c5,0x00c6,0x00c7,0x00c8,0x00c9,0x00ca,0x00cb,0x00cc,0x00cd,0x00ce,0x00cf,
	0x00d0,0x00d1,0x00d2,0x00d3,0x00d4,0x00d5,0x00d6,0x00d7,0x00d8,0x00d9,0x00da,0x00db,0x00dc,0x00dd,0x00de,0x00df,
	0x00e0,0x00e1,0x00e2,0x00e3,0x00e4,0x00e5,0x00e6,0x00e7,0x00e8,0x00e9,0x00ea,0x00eb,0x00ec,0x00ed,0x00ee,0x00ef,
	0x00f0,0x00f1,0x00f2,0x00f3,0x00f4,0x00f5,0x00f6,0x00f7,0x00f8,0x00f9,0x00fa,0x00fb,0x00fc,0x00fd,0x00fe,0x00ff
};
*/

static constexpr char32_t CharsetTableISO8859_15[256] = {
	0x0000,0x0001,0x0002,0x0003,0x0004,0x0005,0x0006,0x0007,0x0008,0x0009,0x000a,0x000b,0x000c,0x000d,0x000e,0x000f,
	0x0010,0x0011,0x0012,0x0013,0x0014,0x0015,0x0016,0x0017,0x0018,0x0019,0x001a,0x001b,0x001c,0x001d,0x001e,0x001f,
	0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
	0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
	0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
	0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
	0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
	0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x007f,
	0x0080,0x0081,0x0082,0x0083,0x0084,0x0085,0x0086,0x0087,0x0088,0x0089,0x008a,0x008b,0x008c,0x008d,0x008e,0x008f,
	0x0090,0x0091,0x0092,0x0093,0x0094,0x0095,0x0096,0x0097,0x0098,0x0099,0x009a,0x009b,0x009c,0x009d,0x009e,0x009f,
	0x00a0,0x00a1,0x00a2,0x00a3,0x20ac,0x00a5,0x0160,0x00a7,0x0161,0x00a9,0x00aa,0x00ab,0x00ac,0x00ad,0x00ae,0x00af,
	0x00b0,0x00b1,0x00b2,0x00b3,0x017d,0x00b5,0x00b6,0x00b7,0x017e,0x00b9,0x00ba,0x00bb,0x0152,0x0153,0x0178,0x00bf,
	0x00c0,0x00c1,0x00c2,0x00c3,0x00c4,0x00c5,0x00c6,0x00c7,0x00c8,0x00c9,0x00ca,0x00cb,0x00cc,0x00cd,0x00ce,0x00cf,
	0x00d0,0x00d1,0x00d2,0x00d3,0x00d4,0x00d5,0x00d6,0x00d7,0x00d8,0x00d9,0x00da,0x00db,0x00dc,0x00dd,0x00de,0x00df,
	0x00e0,0x00e1,0x00e2,0x00e3,0x00e4,0x00e5,0x00e6,0x00e7,0x00e8,0x00e9,0x00ea,0x00eb,0x00ec,0x00ed,0x00ee,0x00ef,
	0x00f0,0x00f1,0x00f2,0x00f3,0x00f4,0x00f5,0x00f6,0x00f7,0x00f8,0x00f9,0x00fa,0x00fb,0x00fc,0x00fd,0x00fe,0x00ff
};

static constexpr char32_t CharsetTableWindows1252[256] = {
	0x0000,0x0001,0x0002,0x0003,0x0004,0x0005,0x0006,0x0007,0x0008,0x0009,0x000a,0x000b,0x000c,0x000d,0x000e,0x000f,
	0x0010,0x0011,0x0012,0x0013,0x0014,0x0015,0x0016,0x0017,0x0018,0x0019,0x001a,0x001b,0x001c,0x001d,0x001e,0x001f,
	0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
	0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
	0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
	0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
	0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
	0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x007f,
	0x20ac,0x0081,0x201a,0x0192,0x201e,0x2026,0x2020,0x2021,0x02c6,0x2030,0x0160,0x2039,0x0152,0x008d,0x017d,0x008f,
	0x0090,0x2018,0x2019,0x201c,0x201d,0x2022,0x2013,0x2014,0x02dc,0x2122,0x0161,0x203a,0x0153,0x009d,0x017e,0x0178,
	0x00a0,0x00a1,0x00a2,0x00a3,0x00a4,0x00a5,0x00a6,0x00a7,0x00a8,0x00a9,0x00aa,0x00ab,0x00ac,0x00ad,0x00ae,0x00af,
	0x00b0,0x00b1,0x00b2,0x00b3,0x00b4,0x00b5,0x00b6,0x00b7,0x00b8,0x00b9,0x00ba,0x00bb,0x00bc,0x00bd,0x00be,0x00bf,
	0x00c0,0x00c1,0x00c2,0x00c3,0x00c4,0x00c5,0x00c6,0x00c7,0x00c8,0x00c9,0x00ca,0x00cb,0x00cc,0x00cd,0x00ce,0x00cf,
	0x00d0,0x00d1,0x00d2,0x00d3,0x00d4,0x00d5,0x00d6,0x00d7,0x00d8,0x00d9,0x00da,0x00db,0x00dc,0x00dd,0x00de,0x00df,
	0x00e0,0x00e1,0x00e2,0x00e3,0x00e4,0x00e5,0x00e6,0x00e7,0x00e8,0x00e9,0x00ea,0x00eb,0x00ec,0x00ed,0x00ee,0x00ef,
	0x00f0,0x00f1,0x00f2,0x00f3,0x00f4,0x00f5,0x00f6,0x00f7,0x00f8,0x00f9,0x00fa,0x00fb,0x00fc,0x00fd,0x00fe,0x00ff
};

static constexpr char32_t CharsetTableCP437[256] = {
	0x0000,0x0001,0x0002,0x0003,0x0004,0x0005,0x0006,0x0007,0x0008,0x0009,0x000a,0x000b,0x000c,0x000d,0x000e,0x000f,
	0x0010,0x0011,0x0012,0x0013,0x0014,0x0015,0x0016,0x0017,0x0018,0x0019,0x001a,0x001b,0x001c,0x001d,0x001e,0x001f,
	0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
	0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
	0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
	0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
	0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
	0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x2302,
	0x00c7,0x00fc,0x00e9,0x00e2,0x00e4,0x00e0,0x00e5,0x00e7,0x00ea,0x00eb,0x00e8,0x00ef,0x00ee,0x00ec,0x00c4,0x00c5,
	0x00c9,0x00e6,0x00c6,0x00f4,0x00f6,0x00f2,0x00fb,0x00f9,0x00ff,0x00d6,0x00dc,0x00a2,0x00a3,0x00a5,0x20a7,0x0192,
	0x00e1,0x00ed,0x00f3,0x00fa,0x00f1,0x00d1,0x00aa,0x00ba,0x00bf,0x2310,0x00ac,0x00bd,0x00bc,0x00a1,0x00ab,0x00bb,
	0x2591,0x2592,0x2593,0x2502,0x2524,0x2561,0x2562,0x2556,0x2555,0x2563,0x2551,0x2557,0x255d,0x255c,0x255b,0x2510,
	0x2514,0x2534,0x252c,0x251c,0x2500,0x253c,0x255e,0x255f,0x255a,0x2554,0x2569,0x2566,0x2560,0x2550,0x256c,0x2567,
	0x2568,0x2564,0x2565,0x2559,0x2558,0x2552,0x2553,0x256b,0x256a,0x2518,0x250c,0x2588,0x2584,0x258c,0x2590,0x2580,
	0x03b1,0x00df,0x0393,0x03c0,0x03a3,0x03c3,0x00b5,0x03c4,0x03a6,0x0398,0x03a9,0x03b4,0x221e,0x03c6,0x03b5,0x2229,
	0x2261,0x00b1,0x2265,0x2264,0x2320,0x2321,0x00f7,0x2248,0x00b0,0x2219,0x00b7,0x221a,0x207f,0x00b2,0x25a0,0x00a0
};


#define C(x) (static_cast<uint8>((x)))

// AMS1 actually only supports ASCII plus the modified control characters and no high chars at all.
// Just default to CP437 for those to keep things simple.
static constexpr char32_t CharsetTableCP437AMS[256] = {
	C(' '),0x0001,0x0002,0x0003,0x00e4,0x0005,0x00e5,0x0007,0x0008,0x0009,0x000a,0x000b,0x000c,0x000d,0x00c4,0x00c5, // differs from CP437
	0x0010,0x0011,0x0012,0x0013,0x00f6,0x0015,0x0016,0x0017,0x0018,0x00d6,0x001a,0x001b,0x001c,0x001d,0x001e,0x001f, // differs from CP437
	0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
	0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
	0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
	0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
	0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
	0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x2302,
	0x00c7,0x00fc,0x00e9,0x00e2,0x00e4,0x00e0,0x00e5,0x00e7,0x00ea,0x00eb,0x00e8,0x00ef,0x00ee,0x00ec,0x00c4,0x00c5,
	0x00c9,0x00e6,0x00c6,0x00f4,0x00f6,0x00f2,0x00fb,0x00f9,0x00ff,0x00d6,0x00dc,0x00a2,0x00a3,0x00a5,0x20a7,0x0192,
	0x00e1,0x00ed,0x00f3,0x00fa,0x00f1,0x00d1,0x00aa,0x00ba,0x00bf,0x2310,0x00ac,0x00bd,0x00bc,0x00a1,0x00ab,0x00bb,
	0x2591,0x2592,0x2593,0x2502,0x2524,0x2561,0x2562,0x2556,0x2555,0x2563,0x2551,0x2557,0x255d,0x255c,0x255b,0x2510,
	0x2514,0x2534,0x252c,0x251c,0x2500,0x253c,0x255e,0x255f,0x255a,0x2554,0x2569,0x2566,0x2560,0x2550,0x256c,0x2567,
	0x2568,0x2564,0x2565,0x2559,0x2558,0x2552,0x2553,0x256b,0x256a,0x2518,0x250c,0x2588,0x2584,0x258c,0x2590,0x2580,
	0x03b1,0x00df,0x0393,0x03c0,0x03a3,0x03c3,0x00b5,0x03c4,0x03a6,0x0398,0x03a9,0x03b4,0x221e,0x03c6,0x03b5,0x2229,
	0x2261,0x00b1,0x2265,0x2264,0x2320,0x2321,0x00f7,0x2248,0x00b0,0x2219,0x00b7,0x221a,0x207f,0x00b2,0x25a0,0x00a0
};

// AMS2: Looking at Velvet Studio's bitmap font (TPIC32.PCX), these appear to be the only supported non-ASCII chars.
static constexpr char32_t CharsetTableCP437AMS2[256] = {
	C(' '),0x00a9,0x221a,0x00b7,C('0'),C('1'),C('2'),C('3'),C('4'),C('5'),C('6'),C('7'),C('8'),C('9'),C('A'),C('B'), // differs from CP437
	C('C'),C('D'),C('E'),C('F'),C(' '),0x00a7,C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '),C(' '), // differs from CP437
	0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,0x0028,0x0029,0x002a,0x002b,0x002c,0x002d,0x002e,0x002f,
	0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,0x0038,0x0039,0x003a,0x003b,0x003c,0x003d,0x003e,0x003f,
	0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,0x0048,0x0049,0x004a,0x004b,0x004c,0x004d,0x004e,0x004f,
	0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,0x0058,0x0059,0x005a,0x005b,0x005c,0x005d,0x005e,0x005f,
	0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,0x0068,0x0069,0x006a,0x006b,0x006c,0x006d,0x006e,0x006f,
	0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,0x0078,0x0079,0x007a,0x007b,0x007c,0x007d,0x007e,0x2302,
	0x00c7,0x00fc,0x00e9,0x00e2,0x00e4,0x00e0,0x00e5,0x00e7,0x00ea,0x00eb,0x00e8,0x00ef,0x00ee,0x00ec,0x00c4,0x00c5,
	0x00c9,0x00e6,0x00c6,0x00f4,0x00f6,0x00f2,0x00fb,0x00f9,0x00ff,0x00d6,0x00dc,0x00a2,0x00a3,0x00a5,0x20a7,0x0192,
	0x00e1,0x00ed,0x00f3,0x00fa,0x00f1,0x00d1,0x00aa,0x00ba,0x00bf,0x2310,0x00ac,0x00bd,0x00bc,0x00a1,0x00ab,0x00bb,
	0x2591,0x2592,0x2593,0x2502,0x2524,0x2561,0x2562,0x2556,0x2555,0x2563,0x2551,0x2557,0x255d,0x255c,0x255b,0x2510,
	0x2514,0x2534,0x252c,0x251c,0x2500,0x253c,0x255e,0x255f,0x255a,0x2554,0x2569,0x2566,0x2560,0x2550,0x256c,0x2567,
	0x2568,0x2564,0x2565,0x2559,0x2558,0x2552,0x2553,0x256b,0x256a,0x2518,0x250c,0x2588,0x2584,0x258c,0x2590,0x2580,
	0x03b1,0x00df,0x0393,0x03c0,0x03a3,0x03c3,0x00b5,0x03c4,0x03a6,0x0398,0x03a9,0x03b4,0x221e,0x03c6,0x03b5,0x2229,
	0x2261,0x00b1,0x2265,0x2264,0x2320,0x2321,0x00f7,0x2248,0x00b0,0x2219,0x00b7,0x221a,0x207f,0x00b2,0x25a0,0x00a0
};

#undef C


#if defined(MPT_COMPILER_QUIRK_NO_WCHAR)
using widechar = char32_t;
using widestring = std::u32string;
static constexpr widechar wide_default_replacement = 0xFFFD;
#else // !MPT_COMPILER_QUIRK_NO_WCHAR
using widechar = wchar_t;
using widestring = std::wstring;
static constexpr widechar wide_default_replacement = L'\uFFFD';
#endif // !MPT_COMPILER_QUIRK_NO_WCHAR


#if MPT_OS_WINDOWS

static bool TestCodePage(UINT cp)
{
	return IsValidCodePage(cp) ? true : false;
}

static bool HasCharset(Charset charset)
{
	bool result = false;
	switch(charset)
	{
#if defined(MPT_ENABLE_CHARSET_LOCALE)
		case Charset::Locale:      result = true; break;
#endif
		case Charset::UTF8:        result = TestCodePage(CP_UTF8); break;
		case Charset::ASCII:       result = TestCodePage(20127);   break;
		case Charset::ISO8859_1:   result = TestCodePage(28591);   break;
		case Charset::ISO8859_15:  result = TestCodePage(28605);   break;
		case Charset::CP437:       result = TestCodePage(437);     break;
		case Charset::Windows1252: result = TestCodePage(1252);    break;
		case Charset::CP437AMS:    result = false; break;
		case Charset::CP437AMS2:   result = false; break;
	}
	return result;
}

static UINT CharsetToCodepage(Charset charset)
{
	switch(charset)
	{
#if defined(MPT_ENABLE_CHARSET_LOCALE)
		case Charset::Locale:      return CP_ACP;  break;
#endif
		case Charset::UTF8:        return CP_UTF8; break;
		case Charset::ASCII:       return 20127;   break;
		case Charset::ISO8859_1:   return 28591;   break;
		case Charset::ISO8859_15:  return 28605;   break;
		case Charset::CP437:       return 437;     break;
		case Charset::CP437AMS:    return 437;     break; // fallback, should not happen
		case Charset::CP437AMS2:   return 437;     break; // fallback, should not happen
		case Charset::Windows1252: return 1252;    break;
	}
	return 0;
}

template<typename Tdststring>
static Tdststring EncodeCodepage(UINT codepage, const widestring &src)
{
	static_assert(sizeof(typename Tdststring::value_type) == sizeof(char));
	static_assert((std::is_same<typename Tdststring::value_type, char>::value));
	Tdststring encoded_string;
	int required_size = WideCharToMultiByte(codepage, 0, src.data(), mpt::saturate_cast<int>(src.size()), nullptr, 0, nullptr, nullptr);
	if(required_size > 0)
	{
		encoded_string.resize(required_size);
		WideCharToMultiByte(codepage, 0, src.data(), mpt::saturate_cast<int>(src.size()), reinterpret_cast<CHAR*>(encoded_string.data()), required_size, nullptr, nullptr);
	}
	return encoded_string;
}

template<typename Tsrcstring>
static widestring DecodeCodepage(UINT codepage, const Tsrcstring &src)
{
	static_assert(sizeof(typename Tsrcstring::value_type) == sizeof(char));
	static_assert((std::is_same<typename Tsrcstring::value_type, char>::value));
	widestring decoded_string;
	int required_size = MultiByteToWideChar(codepage, 0, reinterpret_cast<const CHAR*>(src.data()), mpt::saturate_cast<int>(src.size()), nullptr, 0);
	if(required_size > 0)
	{
		decoded_string.resize(required_size);
		MultiByteToWideChar(codepage, 0, reinterpret_cast<const CHAR*>(src.data()), mpt::saturate_cast<int>(src.size()), decoded_string.data(), required_size);
	}
	return decoded_string;
}

#endif // MPT_OS_WINDOWS


template<typename Tsrcstring>
static widestring From8bit(const Tsrcstring &str, const char32_t (&table)[256], widechar replacement = wide_default_replacement)
{
	widestring res;
	res.reserve(str.length());
	for(std::size_t i = 0; i < str.length(); ++i)
	{
		std::size_t c = static_cast<std::size_t>(static_cast<uint8>(str[i]));
		if(c < std::size(table))
		{
			res.push_back(static_cast<widechar>(table[c]));
		} else
		{
			res.push_back(replacement);
		}
	}
	return res;
}

template<typename Tdststring>
static Tdststring To8bit(const widestring &str, const char32_t (&table)[256], char replacement = '?')
{
	Tdststring res;
	res.reserve(str.length());
	for(std::size_t i = 0; i < str.length(); ++i)
	{
		char32_t c = static_cast<char32_t>(str[i]);
		bool found = false;
		// Try non-control characters first.
		// In cases where there are actual characters mirrored in this range (like in AMS/AMS2 character sets),
		// characters in the common range are preferred this way.
		for(std::size_t x = 0x20; x < std::size(table); ++x)
		{
			if(c == table[x])
			{
				res.push_back(static_cast<typename Tdststring::value_type>(static_cast<uint8>(x)));
				found = true;
				break;
			}
		}
		if(!found)
		{
			// try control characters
			for(std::size_t x = 0x00; x < std::size(table) && x < 0x20; ++x)
			{
				if(c == table[x])
				{
					res.push_back(static_cast<typename Tdststring::value_type>(static_cast<uint8>(x)));
					found = true;
					break;
				}
			}
		}
		if(!found)
		{
			res.push_back(replacement);
		}
	}
	return res;
}

template<typename Tsrcstring>
static widestring FromAscii(const Tsrcstring &str, widechar replacement = wide_default_replacement)
{
	widestring res;
	res.reserve(str.length());
	for(std::size_t i = 0; i < str.length(); ++i)
	{
		uint8 c = str[i];
		if(c <= 0x7f)
		{
			res.push_back(static_cast<widechar>(static_cast<uint32>(c)));
		} else
		{
			res.push_back(replacement);
		}
	}
	return res;
}

template<typename Tdststring>
static Tdststring ToAscii(const widestring &str, char replacement = '?')
{
	Tdststring res;
	res.reserve(str.length());
	for(std::size_t i = 0; i < str.length(); ++i)
	{
		char32_t c = static_cast<char32_t>(str[i]);
		if(c <= 0x7f)
		{
			res.push_back(static_cast<typename Tdststring::value_type>(static_cast<uint8>(c)));
		} else
		{
			res.push_back(replacement);
		}
	}
	return res;
}

template<typename Tsrcstring>
static widestring FromISO_8859_1(const Tsrcstring &str, widechar replacement = wide_default_replacement)
{
	MPT_UNREFERENCED_PARAMETER(replacement);
	widestring res;
	res.reserve(str.length());
	for(std::size_t i = 0; i < str.length(); ++i)
	{
		uint8 c = str[i];
		res.push_back(static_cast<widechar>(static_cast<uint32>(c)));
	}
	return res;
}

template<typename Tdststring>
static Tdststring ToISO_8859_1(const widestring &str, char replacement = '?')
{
	Tdststring res;
	res.reserve(str.length());
	for(std::size_t i = 0; i < str.length(); ++i)
	{
		char32_t c = static_cast<char32_t>(str[i]);
		if(c <= 0xff)
		{
			res.push_back(static_cast<typename Tdststring::value_type>(static_cast<uint8>(c)));
		} else
		{
			res.push_back(replacement);
		}
	}
	return res;
}


#if defined(MPT_ENABLE_CHARSET_LOCALE) && !defined(MPT_LOCALE_ASSUME_CHARSET)

// Note:
//
//  std::codecvt::out in LLVM libc++ does not advance in and out pointers when
// running into a non-convertible character. This can happen when no locale is
// set on FreeBSD or MacOSX. This behaviour violates the C++ standard.
//
//  We apply the following (albeit costly, even on other platforms) work-around:
//  If the conversion errors out and does not advance the pointers at all, we
// retry the conversion with a space character prepended to the string. If it
// still does error out, we retry the whole conversion character by character.
//  This is costly even on other platforms in one single case: The first
// character is an invalid Unicode code point or otherwise not convertible. Any
// following non-convertible characters are not a problem.

static std::wstring LocaleDecode(const std::string &str, const std::locale & locale, wchar_t replacement = L'\uFFFD', int retry = 0, bool * progress = nullptr)
{
	if(str.empty())
	{
		return std::wstring();
	}
	std::vector<wchar_t> out;
	using codecvt_type = std::codecvt<wchar_t, char, std::mbstate_t>;
	std::mbstate_t state = std::mbstate_t();
	const codecvt_type & facet = std::use_facet<codecvt_type>(locale);
	codecvt_type::result result = codecvt_type::partial;
	const char * in_begin = str.data();
	const char * in_end = in_begin + str.size();
	out.resize((in_end - in_begin) * (facet.max_length() + 1));
	wchar_t * out_begin = &(out[0]);
	wchar_t * out_end = &(out[0]) + out.size();
	const char * in_next = nullptr;
	wchar_t * out_next = nullptr;
	do
	{
		if(retry == 2)
		{
			for(;;)
			{
				in_next = nullptr;
				out_next = nullptr;
				result = facet.in(state, in_begin, in_begin + 1, in_next, out_begin, out_end, out_next);
				if(result == codecvt_type::partial && in_next == in_begin + 1)
				{
					in_begin = in_next;
					out_begin = out_next;
					continue;
				} else
				{
					break;
				}
			}
		} else
		{
			in_next = nullptr;
			out_next = nullptr;
			result = facet.in(state, in_begin, in_end, in_next, out_begin, out_end, out_next);
		}
		if(result == codecvt_type::partial || (result == codecvt_type::error && out_next == out_end))
		{
			out.resize(out.size() * 2);
			in_begin = in_next;
			out_begin = &(out[0]) + (out_next - out_begin);
			out_end = &(out[0]) + out.size();
			continue;
		}
		if(retry == 0)
		{
			if(result == codecvt_type::error && in_next == in_begin && out_next == out_begin)
			{
				bool made_progress = true;
				LocaleDecode(std::string(" ") + str, locale, replacement, 1, &made_progress);
				if(!made_progress)
				{
					return LocaleDecode(str, locale, replacement, 2);
				}
			}
		} else if(retry == 1)
		{
			if(result == codecvt_type::error && in_next == in_begin && out_next == out_begin)
			{
				*progress = false;
			} else
			{
				*progress = true;
			}
			return std::wstring();
		}
		if(result == codecvt_type::error)
		{
			++in_next;
			*out_next = replacement;
			++out_next;
		}
		in_begin = in_next;
		out_begin = out_next;
	} while((result == codecvt_type::error && in_next < in_end && out_next < out_end) || (retry == 2 && in_next < in_end));
	return std::wstring(&(out[0]), out_next);
}

static std::string LocaleEncode(const std::wstring &str, const std::locale & locale, char replacement = '?', int retry = 0, bool * progress = nullptr)
{
	if(str.empty())
	{
		return std::string();
	}
	std::vector<char> out;
	using codecvt_type = std::codecvt<wchar_t, char, std::mbstate_t>;
	std::mbstate_t state = std::mbstate_t();
	const codecvt_type & facet = std::use_facet<codecvt_type>(locale);
	codecvt_type::result result = codecvt_type::partial;
	const wchar_t * in_begin = str.data();
	const wchar_t * in_end = in_begin + str.size();
	out.resize((in_end - in_begin) * (facet.max_length() + 1));
	char * out_begin = &(out[0]);
	char * out_end = &(out[0]) + out.size();
	const wchar_t * in_next = nullptr;
	char * out_next = nullptr;
	do
	{
		if(retry == 2)
		{
			for(;;)
			{
				in_next = nullptr;
				out_next = nullptr;
				result = facet.out(state, in_begin, in_begin + 1, in_next, out_begin, out_end, out_next);
				if(result == codecvt_type::partial && in_next == in_begin + 1)
				{
					in_begin = in_next;
					out_begin = out_next;
					continue;
				} else
				{
					break;
				}
			}
		} else
		{
			in_next = nullptr;
			out_next = nullptr;
			result = facet.out(state, in_begin, in_end, in_next, out_begin, out_end, out_next);
		}
		if(result == codecvt_type::partial || (result == codecvt_type::error && out_next == out_end))
		{
			out.resize(out.size() * 2);
			in_begin = in_next;
			out_begin = &(out[0]) + (out_next - out_begin);
			out_end = &(out[0]) + out.size();
			continue;
		}
		if(retry == 0)
		{
			if(result == codecvt_type::error && in_next == in_begin && out_next == out_begin)
			{
				bool made_progress = true;
				LocaleEncode(std::wstring(L" ") + str, locale, replacement, 1, &made_progress);
				if(!made_progress)
				{
					return LocaleEncode(str, locale, replacement, 2);
				}
			}
		} else if(retry == 1)
		{
			if(result == codecvt_type::error && in_next == in_begin && out_next == out_begin)
			{
				*progress = false;
			} else
			{
				*progress = true;
			}
			return std::string();
		}
		if(result == codecvt_type::error)
		{
			++in_next;
			*out_next = replacement;
			++out_next;
		}
		in_begin = in_next;
		out_begin = out_next;
	} while((result == codecvt_type::error && in_next < in_end && out_next < out_end) || (retry == 2 && in_next < in_end));
	return std::string(&(out[0]), out_next);
}

static std::wstring FromLocaleCpp(const std::string &str, wchar_t replacement)
{
	try
	{
		std::locale locale(""); // user locale
		return String::LocaleDecode(str, locale, replacement);
	} MPT_EXCEPTION_CATCH_OUT_OF_MEMORY(e)
	{
		MPT_EXCEPTION_RETHROW_OUT_OF_MEMORY(e);
	} catch(...)
	{
		// nothing
	}
	try
	{
		std::locale locale; // current c++ locale
		return String::LocaleDecode(str, locale, replacement);
	} MPT_EXCEPTION_CATCH_OUT_OF_MEMORY(e)
	{
		MPT_EXCEPTION_RETHROW_OUT_OF_MEMORY(e);
	} catch(...)
	{
		// nothing
	}
	try
	{
		std::locale locale = std::locale::classic(); // "C" locale
		return String::LocaleDecode(str, locale, replacement);
	} MPT_EXCEPTION_CATCH_OUT_OF_MEMORY(e)
	{
		MPT_EXCEPTION_RETHROW_OUT_OF_MEMORY(e);
	} catch(...)
	{
		// nothing
	}
	MPT_ASSERT_NOTREACHED();
	return String::FromAscii<std::string>(str, replacement); // fallback
}

static std::string ToLocaleCpp(const std::wstring &str, char replacement)
{
	try
	{
		std::locale locale(""); // user locale
		return String::LocaleEncode(str, locale, replacement);
	} MPT_EXCEPTION_CATCH_OUT_OF_MEMORY(e)
	{
		MPT_EXCEPTION_RETHROW_OUT_OF_MEMORY(e);
	} catch(...)
	{
		// nothing
	}
	try
	{
		std::locale locale; // current c++ locale
		return String::LocaleEncode(str, locale, replacement);
	} MPT_EXCEPTION_CATCH_OUT_OF_MEMORY(e)
	{
		MPT_EXCEPTION_RETHROW_OUT_OF_MEMORY(e);
	} catch(...)
	{
		// nothing
	}
	try
	{
		std::locale locale = std::locale::classic(); // "C" locale
		return String::LocaleEncode(str, locale, replacement);
	} MPT_EXCEPTION_CATCH_OUT_OF_MEMORY(e)
	{
		MPT_EXCEPTION_RETHROW_OUT_OF_MEMORY(e);
	} catch(...)
	{
		// nothing
	}
	MPT_ASSERT_NOTREACHED();
	return String::ToAscii<std::string>(str, replacement); // fallback
}


template <typename Tsrcstring>
static std::wstring FromLocale(const Tsrcstring &str, wchar_t replacement = L'\uFFFD')
{
	std::string tmp(str.begin(), str.end());
	return FromLocaleCpp(tmp, replacement);
}
template <>
std::wstring FromLocale<std::string>(const std::string &str, wchar_t replacement)
{
	return FromLocaleCpp(str, replacement);
}

template <typename Tdststring>
static Tdststring ToLocale(const std::wstring &str, char replacement = '?')
{
	std::string tmp = ToLocaleCpp(str, replacement);
	return Tdststring(tmp.begin(), tmp.end());
}
template <>
std::string ToLocale(const std::wstring &str, char replacement)
{
	return ToLocaleCpp(str, replacement);
}


#endif // MPT_ENABLE_CHARSET_LOCALE && !MPT_LOCALE_ASSUME_CHARSET

template <typename Tsrcstring>
static widestring FromUTF8(const Tsrcstring &str, widechar replacement = wide_default_replacement)
{
	const Tsrcstring &in = str;

	widestring out;

	// state:
	std::size_t charsleft = 0;
	char32_t ucs4 = 0;

	for ( uint8 c : in ) {

		if ( charsleft == 0 ) {

			if ( ( c & 0x80 ) == 0x00 ) {
				out.push_back( (widechar)c );
			} else if ( ( c & 0xE0 ) == 0xC0 ) {
				ucs4 = c & 0x1F;
				charsleft = 1;
			} else if ( ( c & 0xF0 ) == 0xE0 ) {
				ucs4 = c & 0x0F;
				charsleft = 2;
			} else if ( ( c & 0xF8 ) == 0xF0 ) {
				ucs4 = c & 0x07;
				charsleft = 3;
			} else {
				out.push_back( replacement );
				ucs4 = 0;
				charsleft = 0;
			}

		} else {

			if ( ( c & 0xC0 ) != 0x80 ) {
				out.push_back( replacement );
				ucs4 = 0;
				charsleft = 0;
			}
			ucs4 <<= 6;
			ucs4 |= c & 0x3F;
			charsleft--;

			if ( charsleft == 0 ) {
				if constexpr ( sizeof( widechar ) == 2 ) {
					if ( ucs4 > 0x1fffff ) {
						out.push_back( replacement );
						ucs4 = 0;
						charsleft = 0;
					}
					if ( ucs4 <= 0xffff ) {
						out.push_back( static_cast<widechar>(ucs4) );
					} else {
						uint32 surrogate = static_cast<uint32>(ucs4) - 0x10000;
						uint16 hi_sur = static_cast<uint16>( ( 0x36 << 10 ) | ( (surrogate>>10) & ((1<<10)-1) ) );
						uint16 lo_sur = static_cast<uint16>( ( 0x37 << 10 ) | ( (surrogate>> 0) & ((1<<10)-1) ) );
						out.push_back( hi_sur );
						out.push_back( lo_sur );
					}
				} else {
					out.push_back( static_cast<widechar>( ucs4 ) );
				}
				ucs4 = 0;
			}

		}

	}

	if ( charsleft != 0 ) {
		out.push_back( replacement );
		ucs4 = 0;
		charsleft = 0;
	}

	return out;

}

template <typename Tdststring>
static Tdststring ToUTF8(const widestring &str, char replacement = '?')
{
	const widestring &in = str;

	Tdststring out;

	for ( std::size_t i=0; i<in.length(); i++ ) {

		widechar wc = in[i];

		char32_t ucs4 = 0;
		if constexpr ( sizeof( widechar ) == 2 ) {
			uint16 c = static_cast<uint16>( wc );
			if ( i + 1 < in.length() ) {
				// check for surrogate pair
				uint16 hi_sur = in[i+0];
				uint16 lo_sur = in[i+1];
				if ( hi_sur >> 10 == 0x36 && lo_sur >> 10 == 0x37 ) {
					// surrogate pair
					++i;
					hi_sur &= (1<<10)-1;
					lo_sur &= (1<<10)-1;
					ucs4 = ( static_cast<uint32>(hi_sur) << 10 ) | ( static_cast<uint32>(lo_sur) << 0 );
				} else {
					// no surrogate pair
					ucs4 = static_cast<char32_t>( c );
				}
			} else {
				// no surrogate possible
				ucs4 = static_cast<char32_t>( c );
			}
		} else {
			ucs4 = static_cast<char32_t>( wc );
		}
		
		if ( ucs4 > 0x1fffff ) {
			out.push_back( replacement );
			continue;
		}

		uint8 utf8[6];
		std::size_t numchars = 0;
		for ( numchars = 0; numchars < 6; numchars++ ) {
			utf8[numchars] = ucs4 & 0x3F;
			ucs4 >>= 6;
			if ( ucs4 == 0 ) {
				break;
			}
		}
		numchars++;

		if ( numchars == 1 ) {
			out.push_back( utf8[0] );
			continue;
		}

		if ( numchars == 2 && utf8[numchars-1] == 0x01 ) {
			// generate shortest form
			out.push_back( utf8[0] | 0x40 );
			continue;
		}

		std::size_t charsleft = numchars;
		while ( charsleft > 0 ) {
			if ( charsleft == numchars ) {
				out.push_back( utf8[ charsleft - 1 ] | ( ((1<<numchars)-1) << (8-numchars) ) );
			} else {
				// cppcheck false-positive
				// cppcheck-suppress arrayIndexOutOfBounds
				out.push_back( utf8[ charsleft - 1 ] | 0x80 );
			}
			charsleft--;
		}

	}

	return out;

}


// templated on 8bit strings because of type-safe variants
template<typename Tdststring>
static Tdststring EncodeImpl(Charset charset, const widestring &src)
{
	static_assert(sizeof(typename Tdststring::value_type) == sizeof(char));
	static_assert((std::is_same<typename Tdststring::value_type, char>::value));
	#if defined(MPT_ENABLE_CHARSET_LOCALE)
		#if defined(MPT_LOCALE_ASSUME_CHARSET)
			if(charset == Charset::Locale)
			{
				charset = MPT_LOCALE_ASSUME_CHARSET;
			}
		#endif
	#endif
	#if MPT_OS_WINDOWS
		if(HasCharset(charset))
		{
			return EncodeCodepage<Tdststring>(CharsetToCodepage(charset), src);
		}
	#endif
	switch(charset)
	{
#if defined(MPT_ENABLE_CHARSET_LOCALE)
	#if defined(MPT_LOCALE_ASSUME_CHARSET)
		case Charset::Locale:      MPT_ASSERT_NOTREACHED(); break;
	#else
		case Charset::Locale:      return String::ToLocale<Tdststring>(src); break;
	#endif
#endif
		case Charset::UTF8:        return String::ToUTF8<Tdststring>(src); break;
		case Charset::ASCII:       return String::ToAscii<Tdststring>(src); break;
		case Charset::ISO8859_1:   return String::ToISO_8859_1<Tdststring>(src); break;
		case Charset::ISO8859_15:  return String::To8bit<Tdststring>(src, CharsetTableISO8859_15); break;
		case Charset::CP437:       return String::To8bit<Tdststring>(src, CharsetTableCP437); break;
		case Charset::CP437AMS:    return String::To8bit<Tdststring>(src, CharsetTableCP437AMS); break;
		case Charset::CP437AMS2:   return String::To8bit<Tdststring>(src, CharsetTableCP437AMS2); break;
		case Charset::Windows1252: return String::To8bit<Tdststring>(src, CharsetTableWindows1252); break;
	}
	return Tdststring();
}


// templated on 8bit strings because of type-safe variants
template<typename Tsrcstring>
static widestring DecodeImpl(Charset charset, const Tsrcstring &src)
{
	static_assert(sizeof(typename Tsrcstring::value_type) == sizeof(char));
	static_assert((std::is_same<typename Tsrcstring::value_type, char>::value));
	#if defined(MPT_ENABLE_CHARSET_LOCALE)
		#if defined(MPT_LOCALE_ASSUME_CHARSET)
			if(charset == Charset::Locale)
			{
				charset = MPT_LOCALE_ASSUME_CHARSET;
			}
		#endif
	#endif
	#if MPT_OS_WINDOWS
		if(HasCharset(charset))
		{
			return DecodeCodepage<Tsrcstring>(CharsetToCodepage(charset), src);
		}
	#endif
	switch(charset)
	{
#if defined(MPT_ENABLE_CHARSET_LOCALE)
	#if defined(MPT_LOCALE_ASSUME_CHARSET)
		case Charset::Locale:      MPT_ASSERT_NOTREACHED(); break;
	#else
		case Charset::Locale:      return String::FromLocale<Tsrcstring>(src); break;
	#endif
#endif
		case Charset::UTF8:        return String::FromUTF8<Tsrcstring>(src); break;
		case Charset::ASCII:       return String::FromAscii<Tsrcstring>(src); break;
		case Charset::ISO8859_1:   return String::FromISO_8859_1<Tsrcstring>(src); break;
		case Charset::ISO8859_15:  return String::From8bit<Tsrcstring>(src, CharsetTableISO8859_15); break;
		case Charset::CP437:       return String::From8bit<Tsrcstring>(src, CharsetTableCP437); break;
		case Charset::CP437AMS:    return String::From8bit<Tsrcstring>(src, CharsetTableCP437AMS); break;
		case Charset::CP437AMS2:   return String::From8bit<Tsrcstring>(src, CharsetTableCP437AMS2); break;
		case Charset::Windows1252: return String::From8bit<Tsrcstring>(src, CharsetTableWindows1252); break;
	}
	return widestring();
}


// templated on 8bit strings because of type-safe variants
template<typename Tdststring, typename Tsrcstring>
static Tdststring ConvertImpl(Charset to, Charset from, const Tsrcstring &src)
{
	static_assert(sizeof(typename Tdststring::value_type) == sizeof(char));
	static_assert(sizeof(typename Tsrcstring::value_type) == sizeof(char));
	if(to == from)
	{
		const typename Tsrcstring::value_type * src_beg = src.data();
		const typename Tsrcstring::value_type * src_end = src_beg + src.size();
		return Tdststring(reinterpret_cast<const typename Tdststring::value_type *>(src_beg), reinterpret_cast<const typename Tdststring::value_type *>(src_end));
	}
	return EncodeImpl<Tdststring>(to, DecodeImpl(from, src));
}



} // namespace String


bool IsUTF8(const std::string &str)
{
	return (str == String::EncodeImpl<std::string>(mpt::Charset::UTF8, String::DecodeImpl<std::string>(mpt::Charset::UTF8, str)));
}


#if MPT_WSTRING_CONVERT
std::wstring ToWide(Charset from, const std::string &str)
{
	return String::DecodeImpl(from, str);
}
#if defined(MPT_ENABLE_CHARSET_LOCALE)
std::wstring ToWide(const mpt::lstring &str)
{
	return String::DecodeImpl(Charset::Locale, str);
}
#endif // MPT_ENABLE_CHARSET_LOCALE
#endif

#if MPT_WSTRING_CONVERT
std::string ToCharset(Charset to, const std::wstring &str)
{
	return String::EncodeImpl<std::string>(to, str);
}
#endif
std::string ToCharset(Charset to, Charset from, const std::string &str)
{
	return String::ConvertImpl<std::string>(to, from, str);
}
#if defined(MPT_ENABLE_CHARSET_LOCALE)
std::string ToCharset(Charset to, const mpt::lstring &str)
{
	return String::ConvertImpl<std::string>(to, Charset::Locale, str);
}
#endif // MPT_ENABLE_CHARSET_LOCALE

#if defined(MPT_ENABLE_CHARSET_LOCALE)
#if MPT_WSTRING_CONVERT
mpt::lstring ToLocale(const std::wstring &str)
{
	return String::EncodeImpl<mpt::lstring>(Charset::Locale, str);
}
#endif
mpt::lstring ToLocale(Charset from, const std::string &str)
{
	return String::ConvertImpl<mpt::lstring>(Charset::Locale, from, str);
}
#endif // MPT_ENABLE_CHARSET_LOCALE

#if MPT_OS_WINDOWS
#if MPT_WSTRING_CONVERT
mpt::winstring ToWin(const std::wstring &str)
{
	#ifdef UNICODE
		return str;
	#else
		return ToLocale(str);
	#endif
}
#endif
mpt::winstring ToWin(Charset from, const std::string &str)
{
	#ifdef UNICODE
		return ToWide(from, str);
	#else
		return ToLocale(from, str);
	#endif
}
#if defined(MPT_ENABLE_CHARSET_LOCALE)
mpt::winstring ToWin(const mpt::lstring &str)
{
	#ifdef UNICODE
		return ToWide(str);
	#else
		return str;
	#endif
}
#endif // MPT_ENABLE_CHARSET_LOCALE
#endif // MPT_OS_WINDOWS


#if defined(MPT_WITH_MFC)

CString ToCString(const std::wstring &str)
{
	#ifdef UNICODE
		return str.c_str();
	#else
		return ToCharset(Charset::Locale, str).c_str();
	#endif
}
CString ToCString(Charset from, const std::string &str)
{
	#ifdef UNICODE
		return ToWide(from, str).c_str();
	#else
		return ToCharset(Charset::Locale, from, str).c_str();
	#endif
}
std::wstring ToWide(const CString &str)
{
	#ifdef UNICODE
		return str.GetString();
	#else
		return ToWide(Charset::Locale, str.GetString());
	#endif
}
std::string ToCharset(Charset to, const CString &str)
{
	#ifdef UNICODE
		return ToCharset(to, str.GetString());
	#else
		return ToCharset(to, Charset::Locale, str.GetString());
	#endif
}
#if defined(MPT_ENABLE_CHARSET_LOCALE)
CString ToCString(const mpt::lstring &str)
{
	#ifdef UNICODE
		return ToWide(str).c_str();
	#else
		return str.c_str();
	#endif
}
mpt::lstring ToLocale(const CString &str)
{
	#ifdef UNICODE
		return String::EncodeImpl<mpt::lstring>(Charset::Locale, str.GetString());
	#else
		return str.GetString();
	#endif
}
#endif // MPT_ENABLE_CHARSET_LOCALE
#if MPT_OS_WINDOWS
mpt::winstring ToWin(const CString &str)
{
	return str.GetString();
}
#endif // MPT_OS_WINDOWS

#endif // MPT_WITH_MFC


#if MPT_USTRING_MODE_WIDE
// inline
#else // !MPT_USTRING_MODE_WIDE
#if MPT_WSTRING_CONVERT
mpt::ustring ToUnicode(const std::wstring &str)
{
	return String::EncodeImpl<mpt::ustring>(mpt::Charset::UTF8, str);
}
#endif
mpt::ustring ToUnicode(Charset from, const std::string &str)
{
	return String::ConvertImpl<mpt::ustring>(mpt::Charset::UTF8, from, str);
}
#if defined(MPT_ENABLE_CHARSET_LOCALE)
mpt::ustring ToUnicode(const mpt::lstring &str)
{
	return String::ConvertImpl<mpt::ustring>(mpt::Charset::UTF8, mpt::Charset::Locale, str);
}
#endif // MPT_ENABLE_CHARSET_LOCALE
#if defined(MPT_WITH_MFC)
mpt::ustring ToUnicode(const CString &str)
{
	#ifdef UNICODE
		return String::EncodeImpl<mpt::ustring>(mpt::Charset::UTF8, str.GetString());
	#else // !UNICODE
		return String::ConvertImpl<mpt::ustring, std::string>(mpt::Charset::UTF8, mpt::Charset::Locale, str.GetString());
	#endif // UNICODE
}
#endif // MPT_WITH_MFC
#endif // MPT_USTRING_MODE_WIDE

#if MPT_USTRING_MODE_WIDE
// nothing, std::wstring overloads will catch all stuff
#else // !MPT_USTRING_MODE_WIDE
#if MPT_WSTRING_CONVERT
std::wstring ToWide(const mpt::ustring &str)
{
	return String::DecodeImpl<mpt::ustring>(mpt::Charset::UTF8, str);
}
#endif
std::string ToCharset(Charset to, const mpt::ustring &str)
{
	return String::ConvertImpl<std::string, mpt::ustring>(to, mpt::Charset::UTF8, str);
}
#if defined(MPT_ENABLE_CHARSET_LOCALE)
mpt::lstring ToLocale(const mpt::ustring &str)
{
	return String::ConvertImpl<mpt::lstring, mpt::ustring>(mpt::Charset::Locale, mpt::Charset::UTF8, str);
}
#endif // MPT_ENABLE_CHARSET_LOCALE
#if MPT_OS_WINDOWS
mpt::winstring ToWin(const mpt::ustring &str)
{
	#ifdef UNICODE
		return String::DecodeImpl<mpt::ustring>(mpt::Charset::UTF8, str);
	#else
		return String::ConvertImpl<mpt::lstring, mpt::ustring>(mpt::Charset::Locale, mpt::Charset::UTF8, str);
	#endif
}
#endif // MPT_OS_WINDOWS
#if defined(MPT_WITH_MFC)
CString ToCString(const mpt::ustring &str)
{
	#ifdef UNICODE
		return String::DecodeImpl<mpt::ustring>(mpt::Charset::UTF8, str).c_str();
	#else // !UNICODE
		return String::ConvertImpl<std::string, mpt::ustring>(mpt::Charset::Locale, mpt::Charset::UTF8, str).c_str();
	#endif // UNICODE
}
#endif // MPT_WITH_MFC
#endif // MPT_USTRING_MODE_WIDE





static mpt::Charset CharsetFromCodePage(uint16 codepage, mpt::Charset fallback, bool * isFallback = nullptr)
{
	mpt::Charset result = fallback;
	switch(codepage)
	{
	case 65001:
		result = mpt::Charset::UTF8;
		if(isFallback) *isFallback = false;
		break;
	case 20127:
		result = mpt::Charset::ASCII;
		if(isFallback) *isFallback = false;
		break;
	case 28591:
		result = mpt::Charset::ISO8859_1;
		if(isFallback) *isFallback = false;
		break;
	case 28605:
		result = mpt::Charset::ISO8859_15;
		if(isFallback) *isFallback = false;
		break;
	case 437:
		result = mpt::Charset::CP437;
		if(isFallback) *isFallback = false;
		break;
	case 1252:
		result = mpt::Charset::Windows1252;
		if(isFallback) *isFallback = false;
		break;
	default:
		result = fallback;
		if(isFallback) *isFallback = true;
		break;
	}
	return result;
}

mpt::ustring ToUnicode(uint16 codepage, mpt::Charset fallback, const std::string &str)
{
	#if MPT_OS_WINDOWS
		mpt::ustring result;
		bool noCharsetMatch = true;
		mpt::Charset charset = mpt::CharsetFromCodePage(codepage, fallback, &noCharsetMatch);
		if(noCharsetMatch && mpt::String::TestCodePage(codepage))
		{
			result = mpt::ToUnicode(mpt::String::DecodeCodepage<std::string>(codepage, str));
		} else
		{
			result = mpt::ToUnicode(charset, str);
		}
		return result;
	#else // !MPT_OS_WINDOWS
		return mpt::ToUnicode(mpt::CharsetFromCodePage(codepage, fallback), str);
	#endif // MPT_OS_WINDOWS
}





char ToLowerCaseAscii(char c)
{
	if('A' <= c && c <= 'Z')
	{
		c += 'a' - 'A';
	}
	return c;
}

char ToUpperCaseAscii(char c)
{
	if('a' <= c && c <= 'z')
	{
		c -= 'a' - 'A';
	}
	return c;
}

std::string ToLowerCaseAscii(std::string s)
{
	std::transform(s.begin(), s.end(), s.begin(), static_cast<char(*)(char)>(&mpt::ToLowerCaseAscii));
	return s;
}

std::string ToUpperCaseAscii(std::string s)
{
	std::transform(s.begin(), s.end(), s.begin(), static_cast<char(*)(char)>(&mpt::ToUpperCaseAscii));
	return s;
}

int CompareNoCaseAscii(const char *a, const char *b, std::size_t n)
{
	while(n--)
	{
		unsigned char ac = static_cast<unsigned char>(mpt::ToLowerCaseAscii(*a));
		unsigned char bc = static_cast<unsigned char>(mpt::ToLowerCaseAscii(*b));
		if(ac != bc)
		{
			return ac < bc ? -1 : 1;
		} else if(!ac && !bc)
		{
			return 0;
		}
		++a;
		++b;
	}
	return 0;
}

int CompareNoCaseAscii(std::string_view a, std::string_view b)
{
	for(std::size_t i = 0; i < std::min(a.length(), b.length()); ++i)
	{
		unsigned char ac = static_cast<unsigned char>(mpt::ToLowerCaseAscii(a[i]));
		unsigned char bc = static_cast<unsigned char>(mpt::ToLowerCaseAscii(b[i]));
		if(ac != bc)
		{
			return ac < bc ? -1 : 1;
		} else if(!ac && !bc)
		{
			return 0;
		}
	}
	if(a.length() == b.length())
	{
		return 0;
	}
	return a.length() < b.length() ? -1 : 1;
}

int CompareNoCaseAscii(const std::string &a, const std::string &b)
{
	return CompareNoCaseAscii(std::string_view(a), std::string_view(b));
}


#if defined(MODPLUG_TRACKER)

mpt::ustring ToLowerCase(const mpt::ustring &s)
{
	#if defined(MPT_WITH_MFC)
		#if defined(UNICODE)
			CString tmp = mpt::ToCString(s);
			tmp.MakeLower();
			return mpt::ToUnicode(tmp);
		#else // !UNICODE
			CStringW tmp = mpt::ToWide(s).c_str();
			tmp.MakeLower();
			return mpt::ToUnicode(tmp.GetString());
		#endif // UNICODE
	#else // !MPT_WITH_MFC
		std::wstring ws = mpt::ToWide(s);
		std::transform(ws.begin(), ws.end(), ws.begin(), &std::towlower);
		return mpt::ToUnicode(ws);
	#endif // MPT_WITH_MFC
}

mpt::ustring ToUpperCase(const mpt::ustring &s)
{
	#if defined(MPT_WITH_MFC)
		#if defined(UNICODE)
			CString tmp = mpt::ToCString(s);
			tmp.MakeUpper();
			return mpt::ToUnicode(tmp);
		#else // !UNICODE
			CStringW tmp = mpt::ToWide(s).c_str();
			tmp.MakeUpper();
			return mpt::ToUnicode(tmp.GetString());
		#endif // UNICODE
	#else // !MPT_WITH_MFC
		std::wstring ws = mpt::ToWide(s);
		std::transform(ws.begin(), ws.end(), ws.begin(), &std::towlower);
		return mpt::ToUnicode(ws);
	#endif // MPT_WITH_MFC
}

#endif // MODPLUG_TRACKER



} // namespace mpt



OPENMPT_NAMESPACE_END
