iconv.c

Go to the documentation of this file.

Go to the SVN repository for this file.

/* FreeTDS - Library of routines accessing Sybase and Microsoft databases
 * Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005  Brian Bruns
 * Copyright (C) 2010  Frediano Ziglio
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */
 
/**
 * \file
 * \brief Handle character conversions to/from server
 */
 
#include <config.h>
 
#include <stdarg.h>
#include <stdio.h>
#include <assert.h>
 
#if HAVE_STRING_H
#include <string.h>
#endif /* HAVE_STRING_H */
#if HAVE_ERRNO_H
#include <errno.h>
#endif
 
#include <freetds/tds.h>
#include <freetds/iconv.h>
#include <freetds/bool.h>
#include <freetds/bytes.h>
#if HAVE_ICONV
#include <iconv.h>
#endif
 
#define CHARSIZE(charset) ( ((charset)->min_bytes_per_char == (charset)->max_bytes_per_char )? \
                (charset)->min_bytes_per_char : 0 )
 
 
static int collate2charset(TDSCONNECTION * conn, TDS_UCHAR collate[5]);
static size_t skip_one_input_sequence(iconv_t cd, const TDS_ENCODING * charset, const char **input, size_t * input_size);
static int tds_iconv_info_init(TDSICONV * char_conv, int client_canonic, int server_canonic);
static bool tds_iconv_init(void);
static void _iconv_close(iconv_t * cd);
static void tds_iconv_info_close(TDSICONV * char_conv);
 
 
/**
 * \ingroup libtds
 * \defgroup conv Charset conversion
 * Convert between different charsets.
 */
 
#define TDS_ICONV_ENCODING_TABLES
#include <freetds/encodings.h>
 
/* this will contain real iconv names */
static const char *iconv_names[TDS_VECTOR_SIZE(canonic_charsets)];
static bool iconv_initialized = false;
static const char *ucs2name;
static tds_mutex iconv_mtx = TDS_MUTEX_INITIALIZER;
 
enum
{ POS_ISO1, POS_UTF8, POS_UCS2LE, POS_UCS2BE };
 
static const struct {
    uint32_t len;
    /* this field must be aligned at least to 2 bytes */
    char data[12];
} test_strings[4] = {
    /* same string in required charsets */
    { 4, "Ao\xD3\xE5" },
    { 6, "Ao\xC3\x93\xC3\xA5" },
    { 8, "A\x00o\x000\xD3\x00\xE5\x00" },
    { 8, "\x00" "A\x00o\x000\xD3\x00\xE5" },
};
 
/**
 * Initialize charset searching for UTF-8, UCS-2 and ISO8859-1
 */
static bool
tds_iconv_init(void)
{
    int i;
    iconv_t cd;
 
    /* first entries should be constants */
    assert(strcmp(canonic_charsets[POS_ISO1].name, "ISO-8859-1") == 0);
    assert(strcmp(canonic_charsets[POS_UTF8].name, "UTF-8") == 0);
    assert(strcmp(canonic_charsets[POS_UCS2LE].name, "UCS-2LE") == 0);
    assert(strcmp(canonic_charsets[POS_UCS2BE].name, "UCS-2BE") == 0);
 
    /* fast tests for GNU-iconv */
    cd = tds_sys_iconv_open("ISO-8859-1", "UTF-8");
    if (cd != (iconv_t) -1) {
        iconv_names[POS_ISO1] = "ISO-8859-1";
        iconv_names[POS_UTF8] = "UTF-8";
        tds_sys_iconv_close(cd);
    } else {
 
        /* search names for ISO8859-1 and UTF-8 */
        for (i = 0; iconv_aliases[i].alias; ++i) {
            int j;
 
            if (iconv_aliases[i].canonic != POS_ISO1)
                continue;
            for (j = 0; iconv_aliases[j].alias; ++j) {
                if (iconv_aliases[j].canonic != POS_UTF8)
                    continue;
 
                cd = tds_sys_iconv_open(iconv_aliases[i].alias, iconv_aliases[j].alias);
                if (cd != (iconv_t) -1) {
                    iconv_names[POS_ISO1] = iconv_aliases[i].alias;
                    iconv_names[POS_UTF8] = iconv_aliases[j].alias;
                    tds_sys_iconv_close(cd);
                    break;
                }
            }
            if (iconv_names[POS_ISO1])
                break;
        }
        /* required characters not found !!! */
        if (!iconv_names[POS_ISO1]) {
            tdsdump_log(TDS_DBG_ERROR, "iconv name for ISO-8859-1 not found\n");
            return false;
        }
    }
 
    /* now search for UCS-2 */
    cd = tds_sys_iconv_open(iconv_names[POS_ISO1], "UCS-2LE");
    if (cd != (iconv_t) -1) {
        iconv_names[POS_UCS2LE] = "UCS-2LE";
        tds_sys_iconv_close(cd);
    }
    cd = tds_sys_iconv_open(iconv_names[POS_ISO1], "UCS-2BE");
    if (cd != (iconv_t) -1) {
        iconv_names[POS_UCS2BE] = "UCS-2BE";
        tds_sys_iconv_close(cd);
    }
 
    /* long search needed ?? */
    if (!iconv_names[POS_UCS2LE] || !iconv_names[POS_UCS2BE]) {
        for (i = 0; iconv_aliases[i].alias; ++i) {
            if (strncmp(canonic_charsets[iconv_aliases[i].canonic].name, "UCS-2", 5) != 0)
                continue;
 
            cd = tds_sys_iconv_open(iconv_aliases[i].alias, iconv_names[POS_ISO1]);
            if (cd != (iconv_t) -1) {
                char ib[1];
                char ob[4];
                size_t il, ol;
                ICONV_CONST char *pib;
                char *pob;
                int byte_sequence = 0;
 
                /* try to convert 'A' and check result */
                ib[0] = 0x41;
                pib = ib;
                pob = ob;
                il = 1;
                ol = 4;
                ob[0] = ob[1] = 0;
                if (tds_sys_iconv(cd, &pib, &il, &pob, &ol) != (size_t) - 1) {
                    /* byte order sequence ?? */
                    if (ol == 0) {
                        ob[0] = ob[2];
                        byte_sequence = 1;
                        /* TODO save somewhere */
                    }
 
                    /* save name without sequence (if present) */
                    if (ob[0])
                        il = POS_UCS2LE;
                    else
                        il = POS_UCS2BE;
                    if (!iconv_names[il] || !byte_sequence)
                        iconv_names[il] = iconv_aliases[i].alias;
                }
                tds_sys_iconv_close(cd);
            }
        }
    }
    /* we need a UCS-2 (big endian or little endian) */
    if (!iconv_names[POS_UCS2LE] && !iconv_names[POS_UCS2BE]) {
        tdsdump_log(TDS_DBG_ERROR, "iconv name for UCS-2 not found\n");
        return false;
    }
 
    ucs2name = iconv_names[POS_UCS2LE] ? iconv_names[POS_UCS2LE] : iconv_names[POS_UCS2BE];
 
    for (i = 0; i < 4; ++i)
        tdsdump_log(TDS_DBG_INFO1, "local name for %s is %s\n", canonic_charsets[i].name,
                iconv_names[i] ? iconv_names[i] : "(null)");
 
    /* base conversions checks */
    for (i = 0; i < 4 * 4; ++i) {
        const int from = i / 4;
        const int to = i % 4;
        char ob[16];
        size_t il, ol;
        ICONV_CONST char *pib;
        char *pob;
        size_t res;
 
        if (!iconv_names[from] || !iconv_names[to])
            continue;
        cd = tds_sys_iconv_open(iconv_names[to], iconv_names[from]);
        if (cd == (iconv_t) -1) {
            tdsdump_log(TDS_DBG_ERROR, "iconv_open(%s, %s) failed\n", iconv_names[to], iconv_names[from]);
            return false;
        }
 
        pib = (ICONV_CONST char *) test_strings[from].data;
        il = test_strings[from].len;
        pob = ob;
        ol = sizeof(ob);
        res = tds_sys_iconv(cd, &pib, &il, &pob, &ol);
        tds_sys_iconv_close(cd);
 
        if (res != 0
            || sizeof(ob) - ol != test_strings[to].len
            || memcmp(ob, test_strings[to].data, test_strings[to].len) != 0) {
            tdsdump_log(TDS_DBG_ERROR, "iconv(%s, %s) failed res %d\n", iconv_names[to], iconv_names[from], (int) res);
            tdsdump_log(TDS_DBG_ERROR, "len %d\n", (int) (sizeof(ob) - ol));
            return false;
        }
    }
 
    /* success (it should always occurs) */
    return true;
}
 
/**
 * Get iconv name given canonic
 */
static const char *
tds_set_iconv_name(int charset)
{
    int i;
    iconv_t cd;
    const char *name;
 
        tds_mutex_lock(&iconv_mtx);
    assert(iconv_initialized);
 
    /* try using canonic name and UTF-8 and UCS2 */
    name = canonic_charsets[charset].name;
    cd = tds_sys_iconv_open(iconv_names[POS_UTF8], name);
    if (cd != (iconv_t) -1)
        goto found;
    cd = tds_sys_iconv_open(ucs2name, name);
    if (cd != (iconv_t) -1)
        goto found;
 
    /* try all alternatives */
    for (i = 0; iconv_aliases[i].alias; ++i) {
        if (iconv_aliases[i].canonic != charset)
            continue;
 
        name = iconv_aliases[i].alias;
        cd = tds_sys_iconv_open(iconv_names[POS_UTF8], name);
        if (cd != (iconv_t) -1)
            goto found;
        cd = tds_sys_iconv_open(ucs2name, name);
        if (cd != (iconv_t) -1)
            goto found;
    }
 
    /* charset not found, pretend it's ISO 8859-1 */
    iconv_names[charset] = canonic_charsets[POS_ISO1].name;
        tds_mutex_unlock(&iconv_mtx);
    return NULL;
 
found:
    iconv_names[charset] = name;
    tds_sys_iconv_close(cd);
        tds_mutex_unlock(&iconv_mtx);
    return name;
}
 
static void
tds_iconv_reset(TDSICONV *conv)
{
    /*
     * (min|max)_bytes_per_char can be used to divide
     * so init to safe values
     */
    conv->to.charset.min_bytes_per_char = 1;
    conv->to.charset.max_bytes_per_char = 1;
    conv->from.charset.min_bytes_per_char = 1;
    conv->from.charset.max_bytes_per_char = 1;
 
    conv->to.charset.name = conv->from.charset.name = "";
    conv->to.charset.canonic = conv->from.charset.canonic = 0;
    conv->to.cd = (iconv_t) -1;
    conv->from.cd = (iconv_t) -1;
}
 
/**
 * Allocate iconv stuff
 * \return 0 for success
 */
int
tds_iconv_alloc(TDSCONNECTION * conn)
{
    int i;
    TDSICONV *char_conv;
 
    assert(!conn->char_convs);
    if (!(conn->char_convs = tds_new(TDSICONV *, initial_char_conv_count + 1)))
        return 1;
    char_conv = tds_new0(TDSICONV, initial_char_conv_count);
    if (!char_conv) {
        TDS_ZERO_FREE(conn->char_convs);
        return 1;
    }
    conn->char_conv_count = initial_char_conv_count + 1;
 
    for (i = 0; i < initial_char_conv_count; ++i) {
        conn->char_convs[i] = &char_conv[i];
        tds_iconv_reset(&char_conv[i]);
    }
 
    /* chardata is just a pointer to another iconv info */
    conn->char_convs[initial_char_conv_count] = conn->char_convs[client2server_chardata];
 
    return 0;
}
 
/**
 * \addtogroup conv
 * @{ 
 * Set up the initial iconv conversion descriptors.
 * When the socket is allocated, three TDSICONV structures are attached to iconv.  
 * They have fixed meanings:
 *  \li 0. Client <-> UCS-2 (client2ucs2)
 *  \li 1. Client <-> server single-byte charset (client2server_chardata)
 *
 * Other designs that use less data are possible, but these three conversion needs are 
 * very often needed.  By reserving them, we avoid searching the array for our most common purposes.
 *
 * To solve different iconv names and portability problems FreeTDS maintains 
 * a list of aliases each charset.  
 * 
 * First we discover the names of our minimum required charsets (UTF-8, ISO8859-1 and UCS2).  
 * Later, as and when it's needed, we try to discover others.
 *
 * There is one list of canonic names (GNU iconv names) and two sets of aliases
 * (one for other iconv implementations and another for Sybase). For every
 * canonic charset name we cache the iconv name found during discovery. 
 */
TDSRET
tds_iconv_open(TDSCONNECTION * conn, const char *charset, int use_utf16)
{
    static const char UCS_2LE[] = "UCS-2LE";
    int canonic;
    int canonic_charset = tds_canonical_charset(charset);
    int canonic_env_charset = conn->env.charset ? tds_canonical_charset(conn->env.charset) : -1;
    int fOK;
 
    TDS_ENCODING *client = &conn->char_convs[client2ucs2]->from.charset;
    TDS_ENCODING *server = &conn->char_convs[client2ucs2]->to.charset;
 
    tdsdump_log(TDS_DBG_FUNC, "tds_iconv_open(%p, %s, %d)\n", conn, charset, use_utf16);
 
    /* TDS 5.0 support only UTF-16 encodings */
    if (IS_TDS50(conn))
        use_utf16 = true;
 
    /* initialize */
        tds_mutex_lock(&iconv_mtx);
    if (!iconv_initialized) {
        if (!tds_iconv_init()) {
            tdsdump_log(TDS_DBG_ERROR, "error: tds_iconv_init() failed; "
                           "try using GNU libiconv library\n");
                        tds_mutex_unlock(&iconv_mtx);
            return TDS_FAIL;
        }
        iconv_initialized = true;
    }
        tds_mutex_unlock(&iconv_mtx);
 
    /* 
     * Client <-> UCS-2 (client2ucs2)
     */
    tdsdump_log(TDS_DBG_FUNC, "setting up conversions for client charset \"%s\"\n", charset);
 
    tdsdump_log(TDS_DBG_FUNC, "preparing iconv for \"%s\" <-> \"%s\" conversion\n", charset, UCS_2LE);
 
    fOK = 0;
    if (use_utf16) {
        canonic = TDS_CHARSET_UTF_16LE;
        fOK = tds_iconv_info_init(conn->char_convs[client2ucs2], canonic_charset, canonic);
    }
    if (!fOK) {
        canonic = TDS_CHARSET_UCS_2LE;
        fOK = tds_iconv_info_init(conn->char_convs[client2ucs2], canonic_charset, canonic);
    }
    if (!fOK)
        return TDS_FAIL;
 
    /* 
     * How many UTF-8 bytes we need is a function of what the input character set is.
     * TODO This could definitely be more sophisticated, but it deals with the common case.
     */
    if (client->min_bytes_per_char == 1 && client->max_bytes_per_char == 4 && server->max_bytes_per_char == 1) {
        /* ie client is UTF-8 and server is ISO-8859-1 or variant. */
        client->max_bytes_per_char = 3;
    }
 
    /* 
     * Client <-> server single-byte charset
     * TODO: the server hasn't reported its charset yet, so this logic can't work here.  
     *       not sure what to do about that yet.  
     */
    conn->char_convs[client2server_chardata]->flags = TDS_ENCODING_MEMCPY;
    if (canonic_env_charset >= 0) {
        tdsdump_log(TDS_DBG_FUNC, "preparing iconv for \"%s\" <-> \"%s\" conversion\n", charset, conn->env.charset);
        fOK = tds_iconv_info_init(conn->char_convs[client2server_chardata], canonic_charset, canonic_env_charset);
        if (!fOK)
            return TDS_FAIL;
    } else {
        conn->char_convs[client2server_chardata]->from.charset = canonic_charsets[canonic_charset];
        conn->char_convs[client2server_chardata]->to.charset = canonic_charsets[canonic_charset];
    }
 
    tdsdump_log(TDS_DBG_FUNC, "tds_iconv_open: done\n");
    return TDS_SUCCESS;
}
 
/**
 * Open iconv descriptors to convert between character sets (both directions).
 * 1.  Look up the canonical names of the character sets.
 * 2.  Look up their widths.
 * 3.  Ask iconv to open a conversion descriptor.
 * 4.  Fail if any of the above offer any resistance.  
 * \remarks The charset names written to \a iconv will be the canonical names, 
 *          not necessarily the names passed in. 
 */
static int
tds_iconv_info_init(TDSICONV * char_conv, int client_canonical, int server_canonical)
{
    TDS_ENCODING *client = &char_conv->from.charset;
    TDS_ENCODING *server = &char_conv->to.charset;
 
    assert(char_conv->to.cd == (iconv_t) -1);
    assert(char_conv->from.cd == (iconv_t) -1);
 
    if (client_canonical < 0) {
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv_info_init: client charset name \"%d\" invalid\n", client_canonical);
        return 0;
    }
 
    if (server_canonical < 0) {
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv_info_init: server charset name \"%d\" invalid\n", server_canonical);
        return 0;
    }
 
    *client = canonic_charsets[client_canonical];
    *server = canonic_charsets[server_canonical];
 
    /* special case, same charset, no conversion */
    if (client_canonical == server_canonical) {
        char_conv->to.cd = (iconv_t) -1;
        char_conv->from.cd = (iconv_t) -1;
        char_conv->flags = TDS_ENCODING_MEMCPY;
        return 1;
    }
 
    char_conv->flags = 0;
 
    /* get iconv names */
    if (!iconv_names[client_canonical]) {
        if (!tds_set_iconv_name(client_canonical)) {
            tdsdump_log(TDS_DBG_FUNC, "Charset %d not supported by iconv, using \"%s\" instead\n",
                          client_canonical, iconv_names[client_canonical]);
        }
    }
    
    if (!iconv_names[server_canonical]) {
        if (!tds_set_iconv_name(server_canonical)) {
            tdsdump_log(TDS_DBG_FUNC, "Charset %d not supported by iconv, using \"%s\" instead\n",
                          server_canonical, iconv_names[server_canonical]);
        }
    }
 
    char_conv->to.cd = tds_sys_iconv_open(iconv_names[server_canonical], iconv_names[client_canonical]);
    if (char_conv->to.cd == (iconv_t) -1) {
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv_info_init: cannot convert \"%s\"->\"%s\"\n", client->name, server->name);
    }
 
    char_conv->from.cd = tds_sys_iconv_open(iconv_names[client_canonical], iconv_names[server_canonical]);
    if (char_conv->from.cd == (iconv_t) -1) {
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv_info_init: cannot convert \"%s\"->\"%s\"\n", server->name, client->name);
    }
 
    /* TODO, do some optimizations like UCS2 -> UTF8 min,max = 2,2 (UCS2) and 1,4 (UTF8) */
 
    /* tdsdump_log(TDS_DBG_FUNC, "tds_iconv_info_init: converting \"%s\"->\"%s\"\n", client->name, server->name); */
 
    return 1;
}
 
 
static void
_iconv_close(iconv_t * cd)
{
    static const iconv_t invalid = (iconv_t) -1;
 
    if (*cd != invalid) {
        tds_sys_iconv_close(*cd);
        *cd = invalid;
    }
}
 
static void
tds_iconv_info_close(TDSICONV * char_conv)
{
    _iconv_close(&char_conv->to.cd);
    _iconv_close(&char_conv->from.cd);
}
 
void
tds_iconv_close(TDSCONNECTION * conn)
{
    int i;
 
    for (i = 0; i < conn->char_conv_count; ++i)
        tds_iconv_info_close(conn->char_convs[i]);
}
 
#define CHUNK_ALLOC 4
 
void
tds_iconv_free(TDSCONNECTION * conn)
{
    int i;
 
    if (!conn->char_convs)
        return;
    tds_iconv_close(conn);
 
    free(conn->char_convs[0]);
    for (i = initial_char_conv_count + 1; i < conn->char_conv_count; i += CHUNK_ALLOC)
        free(conn->char_convs[i]);
    TDS_ZERO_FREE(conn->char_convs);
    conn->char_conv_count = 0;
}
 
static void
tds_iconv_err(TDSSOCKET *tds, int err)
{
    if (tds)
        tdserror(tds_get_ctx(tds), tds, err, 0);
}
 
/** 
 * Wrapper around iconv(3).  Same parameters, with slightly different behavior.
 * \param tds state information for the socket and the TDS protocol
 * \param io Enumerated value indicating whether the data are being sent to or received from the server. 
 * \param conv information about the encodings involved, including the iconv(3) conversion descriptors. 
 * \param inbuf address of pointer to the input buffer of data to be converted.  
 * \param inbytesleft address of count of bytes in \a inbuf.
 * \param outbuf address of pointer to the output buffer.  
 * \param outbytesleft address of count of bytes in \a outbuf.
 * \retval number of irreversible conversions performed.  -1 on error, see iconv(3) documentation for 
 * a description of the possible values of \e errno.  
 * \remarks Unlike iconv(3), none of the arguments can be nor point to NULL.  Like iconv(3), all pointers will 
 *      be updated.  Success is signified by a nonnegative return code and \a *inbytesleft == 0.  
 *  If the conversion descriptor in \a iconv is -1 or NULL, \a inbuf is copied to \a outbuf, 
 *  and all parameters updated accordingly. 
 * 
 *  If a character in \a inbuf cannot be converted because no such cbaracter exists in the
 *  \a outbuf character set, we emit messages similar to the ones Sybase emits when it fails such a conversion. 
 *  The message varies depending on the direction of the data.  
 *  On a read error, we emit Msg 2403, Severity 16 (EX_INFO):
 *      "WARNING! Some character(s) could not be converted into client's character set. 
 *          Unconverted bytes were changed to question marks ('?')."
 *  On a write error we emit Msg 2402, Severity 16 (EX_USER):
 *      "Error converting client characters into server's character set. Some character(s) could not be converted."
 *        and return an error code.  Client libraries relying on this routine should reflect an error back to the application.  
 *
 * \todo Check for variable multibyte non-UTF-8 input character set.  
 * \todo Use more robust error message generation.  
 * \todo For reads, cope with \a outbuf encodings that don't have the equivalent of an ASCII '?'.  
 * \todo Support alternative to '?' for the replacement character.  
 */
size_t
tds_iconv(TDSSOCKET * tds, TDSICONV * conv, TDS_ICONV_DIRECTION io,
      const char **inbuf, size_t * inbytesleft, char **outbuf, size_t * outbytesleft)
{
    static const iconv_t invalid = (iconv_t) -1;
    TDSICONVDIR *from = NULL;
    TDSICONVDIR *to = NULL;
 
    iconv_t error_cd = invalid;
 
    char quest_mark[] = "?";    /* best to leave non-const; implementations vary */
    ICONV_CONST char *pquest_mark;
    size_t lquest_mark;
    size_t irreversible;
    size_t one_character;
    bool eilseq_raised = false;
    int conv_errno;
    /* cast away const-ness */
    TDS_ERRNO_MESSAGE_FLAGS *suppress = (TDS_ERRNO_MESSAGE_FLAGS*) &conv->suppress;
 
    assert(inbuf && inbytesleft && outbuf && outbytesleft);
 
    /* if empty there's nothing to return.
     * This fix case with some iconv implementation that does
     * not handle *inbuf == NULL and *inbytesleft == 0 as
     * empty strings
     */
    if (*inbytesleft == 0)
        return 0;
 
    switch (io) {
    case to_server:
        from = &conv->from;
        to = &conv->to;
        break;
    case to_client:
        from = &conv->to;
        to = &conv->from;
        break;
    default:
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv: unable to determine if %d means in or out.  \n", io);
        assert(io == to_server || io == to_client);
        break;
    }
 
    /* silly case, memcpy */
    if (conv->flags & TDS_ENCODING_MEMCPY || to->cd == invalid) {
        size_t len = *inbytesleft < *outbytesleft ? *inbytesleft : *outbytesleft;
 
        memcpy(*outbuf, *inbuf, len);
        conv_errno = *inbytesleft > *outbytesleft ? E2BIG : 0;
        *inbytesleft -= len;
        *outbytesleft -= len;
        *inbuf += len;
        *outbuf += len;
        errno = conv_errno;
        return conv_errno ? (size_t) -1 : 0;
    }
 
    /*
     * Call iconv() as many times as necessary, until we reach the end of input or exhaust output.  
     */
    for (;;) {
        conv_errno = 0;
        irreversible = tds_sys_iconv(to->cd, (ICONV_CONST char **) inbuf, inbytesleft, outbuf, outbytesleft);
 
        /* iconv success, return */
        if (irreversible != (size_t) - 1) {
                        if (irreversible > 0) {
                                eilseq_raised = true;
                        }
            /* here we detect end of conversion and try to reset shift state */
            if (inbuf) {
                /*
                 * if inbuf or *inbuf is NULL iconv reset the shift state.
                 * Note that setting inbytesleft to NULL can cause core so don't do it!
                 */
                inbuf = NULL;
                continue;
            }
            break;
        }
 
        /* save errno, other function could change its value */
        conv_errno = errno;
 
        if (conv_errno == EILSEQ)
            eilseq_raised = true;
 
                if (!eilseq_raised || io != to_client || !inbuf)
            break;
        /* 
         * Invalid input sequence encountered reading from server. 
         * Skip one input sequence, adjusting pointers. 
         */
        one_character = skip_one_input_sequence(to->cd, &from->charset, inbuf, inbytesleft);
 
        if (!one_character)
            break;
 
        /* 
         * To replace invalid input with '?', we have to convert a UTF-8 '?' into the output character set.  
         * In unimaginably weird circumstances, this might be impossible.
         * We use UTF-8 instead of ASCII because some implementations 
         * do not convert singlebyte <-> singlebyte.
         */
        if (error_cd == invalid) {
            error_cd = tds_sys_iconv_open(to->charset.name, iconv_names[POS_UTF8]);
            if (error_cd == invalid) {
                break;  /* what to do? */
            }
        }
 
        lquest_mark = 1;
        pquest_mark = quest_mark;
 
        irreversible = tds_sys_iconv(error_cd, &pquest_mark, &lquest_mark, outbuf, outbytesleft);
 
        if (irreversible == (size_t) - 1)
            break;
 
        if (!*inbytesleft)
            break;
    }
 
    if (eilseq_raised && !suppress->eilseq) {
        /* invalid multibyte input sequence encountered */
        if (io == to_client) {
            if (irreversible == (size_t) - 1) {
                tds_iconv_err(tds, TDSEICONV2BIG);
            } else {
                tds_iconv_err(tds, TDSEICONVI);
                conv_errno = 0;
            }
        } else {
            tds_iconv_err(tds, TDSEICONVO);
        }
        suppress->eilseq = 1;
    }
 
    switch (conv_errno) {
    case EINVAL:        /* incomplete multibyte sequence is encountered */
        if (suppress->einval)
            break;
        /* in chunk conversion this can mean we end a chunk inside a character */
        tds_iconv_err(tds, TDSEICONVAVAIL);
        suppress->einval = 1;
        break;
    case E2BIG:     /* output buffer has no more room */
        if (suppress->e2big)
            break;
        tds_iconv_err(tds, TDSEICONVIU);
        suppress->e2big = 1;
        break;
    default:
        break;
    }
 
    if (error_cd != invalid) {
        tds_sys_iconv_close(error_cd);
    }
 
    errno = conv_errno;
    return irreversible;
}
 
/**
 * Get a iconv info structure, allocate and initialize if needed
 */
TDSICONV *
tds_iconv_get_info(TDSCONNECTION * conn, int canonic_client, int canonic_server)
{
    TDSICONV *info;
    int i;
 
    /* search a charset from already allocated charsets */
    for (i = conn->char_conv_count; --i >= initial_char_conv_count;)
        if (canonic_client == conn->char_convs[i]->from.charset.canonic
            && canonic_server == conn->char_convs[i]->to.charset.canonic)
            return conn->char_convs[i];
 
    /* allocate a new iconv structure */
    if (conn->char_conv_count % CHUNK_ALLOC == ((initial_char_conv_count + 1) % CHUNK_ALLOC)) {
        TDSICONV **p;
        TDSICONV *infos;
 
        infos = tds_new(TDSICONV, CHUNK_ALLOC);
        if (!infos)
            return NULL;
        p = (TDSICONV **) realloc(conn->char_convs, sizeof(TDSICONV *) * (conn->char_conv_count + CHUNK_ALLOC));
        if (!p) {
            free(infos);
            return NULL;
        }
        conn->char_convs = p;
        memset(infos, 0, sizeof(TDSICONV) * CHUNK_ALLOC);
        for (i = 0; i < CHUNK_ALLOC; ++i) {
            conn->char_convs[i + conn->char_conv_count] = &infos[i];
            tds_iconv_reset(&infos[i]);
        }
    }
    info = conn->char_convs[conn->char_conv_count++];
 
    /* init */
    if (tds_iconv_info_init(info, canonic_client, canonic_server))
        return info;
 
    tds_iconv_info_close(info);
    --conn->char_conv_count;
    return NULL;
}
 
TDSICONV *
tds_iconv_get(TDSCONNECTION * conn, const char *client_charset, const char *server_charset)
{
    int canonic_client_charset_num = tds_canonical_charset(client_charset);
    int canonic_server_charset_num = tds_canonical_charset(server_charset);
 
    if (canonic_client_charset_num < 0) {
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv_get: what is charset \"%s\"?\n", client_charset);
        return NULL;
    }
    if (canonic_server_charset_num < 0) {
        tdsdump_log(TDS_DBG_FUNC, "tds_iconv_get: what is charset \"%s\"?\n", server_charset);
        return NULL;
    }
 
    return tds_iconv_get_info(conn, canonic_client_charset_num, canonic_server_charset_num);
}
 
/* change singlebyte conversions according to server */
static void
tds_srv_charset_changed_num(TDSCONNECTION * conn, int canonic_charset_num)
{
    TDSICONV *char_conv = conn->char_convs[client2server_chardata];
 
    if (IS_TDS7_PLUS(conn) && canonic_charset_num == TDS_CHARSET_ISO_8859_1)
        canonic_charset_num = TDS_CHARSET_CP1252;
 
    tdsdump_log(TDS_DBG_FUNC, "setting server single-byte charset to \"%s\"\n", canonic_charsets[canonic_charset_num].name);
 
    if (canonic_charset_num == char_conv->to.charset.canonic)
        return;
 
    /* find and set conversion */
    char_conv = tds_iconv_get_info(conn, conn->char_convs[client2ucs2]->from.charset.canonic, canonic_charset_num);
    if (char_conv)
        conn->char_convs[client2server_chardata] = char_conv;
}
 
void
tds_srv_charset_changed(TDSCONNECTION * conn, const char *charset)
{
    int n = tds_canonical_charset(charset);
 
    /* ignore request to change to unknown charset */
    if (n < 0) {
        tdsdump_log(TDS_DBG_FUNC, "tds_srv_charset_changed: what is charset \"%s\"?\n", charset);
        return;
    }
 
    tds_srv_charset_changed_num(conn, n);
}
 
/* change singlebyte conversions according to server */
void
tds7_srv_charset_changed(TDSCONNECTION * conn, TDS_UCHAR collation[5])
{
    tds_srv_charset_changed_num(conn, collate2charset(conn, collation));
}
 
/**
 * Move the input sequence pointer to the next valid position.
 * Used when an input character cannot be converted.  
 * \returns number of bytes to skip.
 */
/* FIXME possible buffer reading overflow ?? */
static size_t
skip_one_input_sequence(iconv_t cd, const TDS_ENCODING * charset, const char **input, size_t * input_size)
{
    unsigned charsize = CHARSIZE(charset);
    char ib[16];
    char ob[16];
    ICONV_CONST char *pib;
    char *pob;
    size_t il, ol, l;
    iconv_t cd2;
 
 
    /* usually fixed size and UTF-8 do not have state, so do not reset it */
    if (charsize)
        goto skip_charsize;
 
    if (0 == strcmp(charset->name, "UTF-8")) {
        /*
         * Deal with UTF-8.  
         * bytes | bits | representation
         *     1 |    7 | 0vvvvvvv
         *     2 |   11 | 110vvvvv 10vvvvvv
         *     3 |   16 | 1110vvvv 10vvvvvv 10vvvvvv
         *     4 |   21 | 11110vvv 10vvvvvv 10vvvvvv 10vvvvvv
         */
        int c = **input;
 
        c = c & (c >> 1);
        do {
            ++charsize;
        } while ((c <<= 1) & 0x80);
        goto skip_charsize;
    }
 
    /* handle state encoding */
 
    /* extract state from iconv */
    pob = ib;
    ol = sizeof(ib);
    tds_sys_iconv(cd, NULL, NULL, &pob, &ol);
 
    /* init destination conversion */
    /* TODO use largest fixed size for this platform */
    cd2 = tds_sys_iconv_open("UCS-4", charset->name);
    if (cd2 == (iconv_t) -1)
        return 0;
 
    /* add part of input */
    il = ol;
    if (il > *input_size)
        il = *input_size;
    l = sizeof(ib) - ol;
    memcpy(ib + l, *input, il);
    il += l;
 
    /* translate a single character */
    pib = ib;
    pob = ob;
    /* TODO use size of largest fixed charset */
    ol = 4;
    tds_sys_iconv(cd2, &pib, &il, &pob, &ol);
 
    /* adjust input */
    l = (pib - ib) - l;
    *input += l;
    *input_size -= l;
 
    /* extract state */
    pob = ib;
    ol = sizeof(ib);
    tds_sys_iconv(cd, NULL, NULL, &pob, &ol);
 
    /* set input state */
    pib = ib;
    il = sizeof(ib) - ol;
    pob = ob;
    ol = sizeof(ob);
    tds_sys_iconv(cd, &pib, &il, &pob, &ol);
 
    tds_sys_iconv_close(cd2);
 
    if (l != 0)
        return l;
 
    /* last blindly attempt, skip minimum bytes */
    charsize = charset->min_bytes_per_char;
 
    /* fall through */
 
skip_charsize:
    if (charsize > *input_size)
        return 0;
    *input += charsize;
    *input_size -= charsize;
    return charsize;
}
 
#include <freetds/charset_lookup.h>
 
/**
 * Determine canonical iconv character set.
 * \returns canonical position, or -1 if lookup failed.
 * \remarks Returned name can be used in bytes_per_char(), above.
 */
int
tds_canonical_charset(const char *charset_name)
{
    const struct charset_alias *c = charset_lookup(charset_name, strlen(charset_name));
    return c ? c->canonic : -1;
}
 
/**
 * Determine canonical iconv character set name.  
 * \returns canonical name, or NULL if lookup failed.
 * \remarks Returned name can be used in bytes_per_char(), above.
 */
const char *
tds_canonical_charset_name(const char *charset_name)
{
    int res;
 
    /* get numeric pos */
    res = tds_canonical_charset(charset_name);
    if (res >= 0)
        return canonic_charsets[res].name;
 
    return charset_name;    /* hope for the best */
}
 
static int
collate2charset(TDSCONNECTION * conn, TDS_UCHAR collate[5])
{
    int cp = 0;
    const int sql_collate = collate[4];
    /* extract 16 bit of LCID (it's 20 bits but higher 4 are just variations) */
    const int lcid = TDS_GET_UA2LE(collate);
 
    /* starting with bit 20 (little endian, so 3rd byte bit 4) there are 8 bits:
     * fIgnoreCase fIgnoreAccent fIgnoreKana fIgnoreWidth fBinary fBinary2 fUTF8 FRESERVEDBIT
     * so fUTF8 is on the 4th byte bit 2 */
    if ((collate[3] & 0x4) != 0 && IS_TDS74_PLUS(conn))
        return TDS_CHARSET_UTF_8;
 
    /*
     * The table from the MSQLServer reference "Windows Collation Designators" 
     * and from " NLS Information for Microsoft Windows XP".
     *
     * See also https://go.microsoft.com/fwlink/?LinkId=119987 [MSDN-SQLCollation]
     */
 
    switch (sql_collate) {
    case 30:        /* SQL_Latin1_General_CP437_BIN */
    case 31:        /* SQL_Latin1_General_CP437_CS_AS */
    case 32:        /* SQL_Latin1_General_CP437_CI_AS */
    case 33:        /* SQL_Latin1_General_Pref_CP437_CI_AS */
    case 34:        /* SQL_Latin1_General_CP437_CI_AI */
        return TDS_CHARSET_CP437;
    case 40:        /* SQL_Latin1_General_CP850_BIN */
    case 41:        /* SQL_Latin1_General_CP850_CS_AS */
    case 42:        /* SQL_Latin1_General_CP850_CI_AS */
    case 43:        /* SQL_Latin1_General_Pref_CP850_CI_AS */
    case 44:        /* SQL_Latin1_General_CP850_CI_AI */
    case 49:        /* SQL_1xCompat_CP850_CI_AS */
    case 55:        /* SQL_AltDiction_CP850_CS_AS */
    case 56:        /* SQL_AltDiction_Pref_CP850_CI_AS */
    case 57:        /* SQL_AltDiction_CP850_CI_AI */
    case 58:        /* SQL_Scandinavian_Pref_CP850_CI_AS */
    case 59:        /* SQL_Scandinavian_CP850_CS_AS */
    case 60:        /* SQL_Scandinavian_CP850_CI_AS */
    case 61:        /* SQL_AltDiction_CP850_CI_AS */
        return TDS_CHARSET_CP850;
    case 80:        /* SQL_Latin1_General_1250_BIN */
    case 81:        /* SQL_Latin1_General_CP1250_CS_AS */
    case 82:        /* SQL_Latin1_General_CP1250_CI_AS */
        return TDS_CHARSET_CP1250;
    case 105:       /* SQL_Latin1_General_CP1251_CS_AS */
    case 106:       /* SQL_Latin1_General_CP1251_CI_AS */
        return TDS_CHARSET_CP1251;
    case 113:       /* SQL_Latin1_General_CP1253_CS_AS */
    case 114:       /* SQL_Latin1_General_CP1253_CI_AS */
    case 120:       /* SQL_MixDiction_CP1253_CS_AS */
    case 121:       /* SQL_AltDiction_CP1253_CS_AS */
    case 122:       /* SQL_AltDiction2_CP1253_CS_AS */
    case 124:       /* SQL_Latin1_General_CP1253_CI_AI */
        return TDS_CHARSET_CP1253;
    case 137:       /* SQL_Latin1_General_CP1255_CS_AS */
    case 138:       /* SQL_Latin1_General_CP1255_CI_AS */
        return TDS_CHARSET_CP1255;
    case 145:       /* SQL_Latin1_General_CP1256_CS_AS */
    case 146:       /* SQL_Latin1_General_CP1256_CI_AS */
        return TDS_CHARSET_CP1256;
    case 153:       /* SQL_Latin1_General_CP1257_CS_AS */
    case 154:       /* SQL_Latin1_General_CP1257_CI_AS */
        return TDS_CHARSET_CP1257;
    }
 
    switch (lcid) {
    case 0x405:
    case 0x40e:     /* 0x1040e */
    case 0x415:
    case 0x418:
    case 0x41a:
    case 0x41b:
    case 0x41c:
    case 0x424:
    case 0x442:
    case 0x81a:
    case 0x104e:        /* ?? */
    case 0x141a:
        cp = TDS_CHARSET_CP1250;
        break;
    case 0x402:
    case 0x419:
    case 0x422:
    case 0x423:
    case 0x42f:
    case 0x43f:
    case 0x440:
    case 0x444:
    case 0x450:
    case 0x82c:
    case 0x843:
    case 0xc1a:
    case 0x46d:
    case 0x201a:
    case 0x485:
        cp = TDS_CHARSET_CP1251;
        break;
    case 0x1007:
    case 0x1009:
    case 0x100a:
    case 0x100c:
    case 0x1407:
    case 0x1409:
    case 0x140a:
    case 0x140c:
    case 0x1809:
    case 0x180a:
    case 0x180c:
    case 0x1c09:
    case 0x1c0a:
    case 0x2009:
    case 0x200a:
    case 0x2409:
    case 0x240a:
    case 0x2809:
    case 0x280a:
    case 0x2c09:
    case 0x2c0a:
    case 0x3009:
    case 0x300a:
    case 0x3409:
    case 0x340a:
    case 0x380a:
    case 0x3c0a:
    case 0x400a:
    case 0x403:
    case 0x406:
    case 0x417:
    case 0x42e:
    case 0x43b:
    case 0x452:
    case 0x462:
    case 0x47a:
    case 0x47c:
    case 0x47e:
    case 0x483:
    case 0x407:     /* 0x10407 */
    case 0x409:
    case 0x40a:
    case 0x40b:
    case 0x40c:
    case 0x40f:
    case 0x410:
    case 0x413:
    case 0x414:
    case 0x416:
    case 0x41d:
    case 0x421:
    case 0x42d:
    case 0x436:
    case 0x437:     /* 0x10437 */
    case 0x438:
        /*case 0x439:  ??? Unicode only */
    case 0x43e:
    case 0x440a:
    case 0x441:
    case 0x456:
    case 0x480a:
    case 0x4c0a:
    case 0x500a:
    case 0x807:
    case 0x809:
    case 0x80a:
    case 0x80c:
    case 0x810:
    case 0x813:
    case 0x814:
    case 0x816:
    case 0x81d:
    case 0x83b:
    case 0x83e:
    case 0x85f:
    case 0xc07:
    case 0xc09:
    case 0xc0a:
    case 0xc0c:
        cp = TDS_CHARSET_CP1252;
        break;
    case 0x408:
        cp = TDS_CHARSET_CP1253;
        break;
    case 0x41f:
    case 0x42c:
    case 0x443:
        cp = TDS_CHARSET_CP1254;
        break;
    case 0x40d:
        cp = TDS_CHARSET_CP1255;
        break;
    case 0x1001:
    case 0x1401:
    case 0x1801:
    case 0x1c01:
    case 0x2001:
    case 0x2401:
    case 0x2801:
    case 0x2c01:
    case 0x3001:
    case 0x3401:
    case 0x3801:
    case 0x3c01:
    case 0x4001:
    case 0x401:
    case 0x480:
    case 0x420:
    case 0x429:
    case 0x48c:
    case 0x801:
    case 0xc01:
        cp = TDS_CHARSET_CP1256;
        break;
    case 0x425:
    case 0x426:
    case 0x427:
    case 0x827:     /* ?? */
        cp = TDS_CHARSET_CP1257;
        break;
    case 0x42a:
        cp = TDS_CHARSET_CP1258;
        break;
    case 0x41e:
        cp = TDS_CHARSET_CP874;
        break;
    case 0x411:     /* 0x10411 */
        cp = TDS_CHARSET_CP932;
        break;
    case 0x1004:
    case 0x804:     /* 0x20804 */
        cp = TDS_CHARSET_GB18030;
        break;
    case 0x412:     /* 0x10412 */
        cp = TDS_CHARSET_CP949;
        break;
    case 0x1404:
    case 0x404:     /* 0x30404 */
    case 0xc04:
        cp = TDS_CHARSET_CP950;
        break;
    default:
        cp = TDS_CHARSET_CP1252;
    }
 
    return cp;
}
 
/**
 * Get iconv information from a LCID (to support different column encoding under MSSQL2K)
 */
TDSICONV *
tds_iconv_from_collate(TDSCONNECTION * conn, TDS_UCHAR collate[5])
{
    int canonic_charset = collate2charset(conn, collate);
 
    /* same as client (usually this is true, so this improve performance) ? */
    if (conn->char_convs[client2server_chardata]->to.charset.canonic == canonic_charset)
        return conn->char_convs[client2server_chardata];
 
    return tds_iconv_get_info(conn, conn->char_convs[client2ucs2]->from.charset.canonic, canonic_charset);
}
 
/** @} */