view lib/unistr/u8-mbtouc-unsafe.c @ 13864:83949b090c98

unistr/u8-mbtouc: Improve handling of ill-formed UTF-8 input. * lib/unistr/u8-mbtouc.c (u8_mbtouc): For an invalid multibyte character, return the number of bytes that belong together, not always 1. * lib/unistr/u8-mbtouc-unsafe.c (u8_mbtouc_unsafe): Likewise. * lib/unistr/u8-mbtouc-aux.c (u8_mbtouc_aux): Likewise. * lib/unistr/u8-mbtouc-unsafe-aux.c (u8_mbtouc_unsafe_aux): Likewise. * lib/unistr/u8-mbsnlen.c (u8_mbsnlen): Use u8_mbtouc to determine the number of bytes of an invalid character. * tests/unistr/test-u8-mbtouc.c (test_safe_function): New function. (main): Invoke it. * tests/unistr/test-u8-mbtouc.h (test_function): Update two test results. * tests/unistr/test-u8-mbsnlen.c (main): Test various kinds of malformed byte sequences. * modules/unistr/u8-mbtouc (configure.ac): Bump version number. * modules/unistr/u8-mbtouc-unsafe (configure.ac): Likewise. * modules/unistr/u8-mbsnlen (configure.ac): Likewise. Reported by Ben Pfaff and Paolo Bonzini.
author Bruno Haible <bruno@clisp.org>
date Sat, 13 Nov 2010 19:43:06 +0100
parents c2cbabec01dd
children 97fc9a21a8fb
line wrap: on
line source

/* Look at first character in UTF-8 string.
   Copyright (C) 1999-2002, 2006-2007, 2009-2010 Free Software Foundation, Inc.
   Written by Bruno Haible <bruno@clisp.org>, 2001.

   This program is free software: you can redistribute it and/or modify it
   under the terms of the GNU Lesser General Public License as published
   by the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

#include <config.h>

#if defined IN_LIBUNISTRING
/* Tell unistr.h to declare u8_mbtouc_unsafe as 'extern', not
   'static inline'.  */
# include "unistring-notinline.h"
#endif

/* Specification.  */
#include "unistr.h"

#if !HAVE_INLINE

int
u8_mbtouc_unsafe (ucs4_t *puc, const uint8_t *s, size_t n)
{
  uint8_t c = *s;

  if (c < 0x80)
    {
      *puc = c;
      return 1;
    }
  else if (c >= 0xc2)
    {
      if (c < 0xe0)
        {
          if (n >= 2)
            {
#if CONFIG_UNICODE_SAFETY
              if ((s[1] ^ 0x80) < 0x40)
#endif
                {
                  *puc = ((unsigned int) (c & 0x1f) << 6)
                         | (unsigned int) (s[1] ^ 0x80);
                  return 2;
                }
#if CONFIG_UNICODE_SAFETY
              /* invalid multibyte character */
#endif
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return 1;
            }
        }
      else if (c < 0xf0)
        {
          if (n >= 3)
            {
#if CONFIG_UNICODE_SAFETY
              if ((s[1] ^ 0x80) < 0x40)
                {
                  if ((s[2] ^ 0x80) < 0x40)
                    {
                      if ((c >= 0xe1 || s[1] >= 0xa0)
                          && (c != 0xed || s[1] < 0xa0))
#endif
                        {
                          *puc = ((unsigned int) (c & 0x0f) << 12)
                                 | ((unsigned int) (s[1] ^ 0x80) << 6)
                                 | (unsigned int) (s[2] ^ 0x80);
                          return 3;
                        }
#if CONFIG_UNICODE_SAFETY
                      /* invalid multibyte character */
                      *puc = 0xfffd;
                      return 3;
                    }
                  /* invalid multibyte character */
                  *puc = 0xfffd;
                  return 2;
                }
              /* invalid multibyte character */
#endif
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              if (n == 1 || (s[1] ^ 0x80) >= 0x40)
                return 1;
              else
                return 2;
            }
        }
      else if (c < 0xf8)
        {
          if (n >= 4)
            {
#if CONFIG_UNICODE_SAFETY
              if ((s[1] ^ 0x80) < 0x40)
                {
                  if ((s[2] ^ 0x80) < 0x40)
                    {
                      if ((s[3] ^ 0x80) < 0x40)
                        {
                          if ((c >= 0xf1 || s[1] >= 0x90)
#if 1
                              && (c < 0xf4 || (c == 0xf4 && s[1] < 0x90))
#endif
                             )
#endif
                            {
                              *puc = ((unsigned int) (c & 0x07) << 18)
                                     | ((unsigned int) (s[1] ^ 0x80) << 12)
                                     | ((unsigned int) (s[2] ^ 0x80) << 6)
                                     | (unsigned int) (s[3] ^ 0x80);
                              return 4;
                            }
#if CONFIG_UNICODE_SAFETY
                          /* invalid multibyte character */
                          *puc = 0xfffd;
                          return 4;
                        }
                      /* invalid multibyte character */
                      *puc = 0xfffd;
                      return 3;
                    }
                  /* invalid multibyte character */
                  *puc = 0xfffd;
                  return 2;
                }
              /* invalid multibyte character */
#endif
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              if (n == 1 || (s[1] ^ 0x80) >= 0x40)
                return 1;
              else if (n == 2 || (s[2] ^ 0x80) >= 0x40)
                return 2;
              else
                return 3;
            }
        }
#if 0
      else if (c < 0xfc)
        {
          if (n >= 5)
            {
#if CONFIG_UNICODE_SAFETY
              if ((s[1] ^ 0x80) < 0x40)
                {
                  if ((s[2] ^ 0x80) < 0x40)
                    {
                      if ((s[3] ^ 0x80) < 0x40)
                        {
                          if ((s[4] ^ 0x80) < 0x40)
                            {
                              if (c >= 0xf9 || s[1] >= 0x88)
#endif
                                {
                                  *puc = ((unsigned int) (c & 0x03) << 24)
                                         | ((unsigned int) (s[1] ^ 0x80) << 18)
                                         | ((unsigned int) (s[2] ^ 0x80) << 12)
                                         | ((unsigned int) (s[3] ^ 0x80) << 6)
                                         | (unsigned int) (s[4] ^ 0x80);
                                  return 5;
                                }
#if CONFIG_UNICODE_SAFETY
                              /* invalid multibyte character */
                              *puc = 0xfffd;
                              return 5;
                            }
                          /* invalid multibyte character */
                          *puc = 0xfffd;
                          return 4;
                        }
                      /* invalid multibyte character */
                      *puc = 0xfffd;
                      return 3;
                    }
                  /* invalid multibyte character */
                  return 2;
                }
              /* invalid multibyte character */
#endif
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
      else if (c < 0xfe)
        {
          if (n >= 6)
            {
#if CONFIG_UNICODE_SAFETY
              if ((s[1] ^ 0x80) < 0x40)
                {
                  if ((s[2] ^ 0x80) < 0x40)
                    {
                      if ((s[3] ^ 0x80) < 0x40)
                        {
                          if ((s[4] ^ 0x80) < 0x40)
                            {
                              if ((s[5] ^ 0x80) < 0x40)
                                {
                                  if (c >= 0xfd || s[1] >= 0x84)
#endif
                                    {
                                      *puc = ((unsigned int) (c & 0x01) << 30)
                                             | ((unsigned int) (s[1] ^ 0x80) << 24)
                                             | ((unsigned int) (s[2] ^ 0x80) << 18)
                                             | ((unsigned int) (s[3] ^ 0x80) << 12)
                                             | ((unsigned int) (s[4] ^ 0x80) << 6)
                                             | (unsigned int) (s[5] ^ 0x80);
                                      return 6;
                                    }
#if CONFIG_UNICODE_SAFETY
                                  /* invalid multibyte character */
                                  *puc = 0xfffd;
                                  return 6;
                                }
                              /* invalid multibyte character */
                              *puc = 0xfffd;
                              return 5;
                            }
                          /* invalid multibyte character */
                          *puc = 0xfffd;
                          return 4;
                        }
                      /* invalid multibyte character */
                      *puc = 0xfffd;
                      return 3;
                    }
                  /* invalid multibyte character */
                  return 2;
                }
              /* invalid multibyte character */
#endif
            }
          else
            {
              /* incomplete multibyte character */
              *puc = 0xfffd;
              return n;
            }
        }
#endif
    }
  /* invalid multibyte character */
  *puc = 0xfffd;
  return 1;
}

#endif