regex.c

Go to the documentation of this file.

/* Extended regular expression matching and search library.
   Copyright (C) 1993, 94, 95, 96, 97, 98 Free Software Foundation, Inc.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Library General Public License as
   published by the Free Software Foundation; either version 2 of the
   License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Library General Public License for more details.

   You should have received a copy of the GNU Library General Public
   License along with the GNU C Library; see the file LGPL.  If not,
   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
   Boston, MA 02111-1307, USA.  */
/* Multi-byte extension added May, 1993 by t^2 (Takahiro Tanimoto)
   Last change: May 21, 1993 by t^2  */
/* removed gapped buffer support, multiple syntax support by matz <matz@nts.co.jp> */
/* Perl5 extension added by matz <matz@caelum.co.jp> */
/* UTF-8 extension added Jan 16 1999 by Yoshida Masato  <yoshidam@tau.bekkoame.ne.jp> */

#include "config.h"

#ifdef HAVE_STRING_H
# include <string.h>
#else
# include <strings.h>
#endif

/* We write fatal error messages on standard error.  */
#include <stdio.h>

/* isalpha(3) etc. are used for the character classes.  */
#include <ctype.h>
#include <sys/types.h>

#ifndef PARAMS
# if defined __GNUC__ || (defined __STDC__ && __STDC__)
#  define PARAMS(args) args
# else
#  define PARAMS(args) ()
# endif  /* GCC.  */
#endif  /* Not PARAMS.  */

#if defined(STDC_HEADERS)
# include <stddef.h>
#else
/* We need this for `regex.h', and perhaps for the Emacs include files.  */
# include <sys/types.h>
#endif

#if !defined(__STDC__) && !defined(_MSC_VER)
# define volatile
#endif

#ifdef HAVE_PROTOTYPES
# define args args
#else
# define args ()
#endif

#ifdef RUBY_PLATFORM
#include "defines.h"

# define RUBY
extern int rb_prohibit_interrupt;
extern int rb_trap_pending;
void rb_trap_exec (void);

# define CHECK_INTS do {\
    if (!rb_prohibit_interrupt) {\
        if (rb_trap_pending) rb_trap_exec();\
    }\
} while (0)
#endif

/* Make alloca work the best possible way.  */
#ifdef __GNUC__
# ifndef atarist
#  ifndef alloca
#   define alloca __builtin_alloca
#  endif
# endif /* atarist */
#else
# if defined(HAVE_ALLOCA_H)
#  include <alloca.h>
# elif !defined(alloca)
char *alloca();
# endif
#endif /* __GNUC__ */

#ifdef _AIX
#pragma alloca
#endif

#ifdef HAVE_STRING_H
# include <string.h>
#else
# include <strings.h>
#endif

#ifdef C_ALLOCA
#define FREE_VARIABLES() alloca(0)
#else
#define FREE_VARIABLES()
#endif

#define FREE_AND_RETURN_VOID(stackb)   do {                             \
  FREE_VARIABLES();                                                     \
  if (stackb != stacka) xfree(stackb);                                  \
  return;                                                               \
} while(0)

#define FREE_AND_RETURN(stackb,val)    do {                             \
  FREE_VARIABLES();                                                     \
  if (stackb != stacka) xfree(stackb);                                  \
  return(val);                                                          \
} while(0)

#define DOUBLE_STACK(type) do {                                         \
  type *stackx;                                                         \
  unsigned int xlen = stacke - stackb;                                  \
  if (stackb == stacka) {                                               \
    stackx = (type*)xmalloc(2 * xlen * sizeof(type));                   \
    memcpy(stackx, stackb, xlen * sizeof (type));                       \
  }                                                                     \
  else {                                                                \
    stackx = (type*)xrealloc(stackb, 2 * xlen * sizeof(type));          \
  }                                                                     \
  /* Rearrange the pointers. */                                         \
  stackp = stackx + (stackp - stackb);                                  \
  stackb = stackx;                                                      \
  stacke = stackb + 2 * xlen;                                           \
} while (0)

#define RE_TALLOC(n,t)  ((t*)alloca((n)*sizeof(t)))
#define TMALLOC(n,t)    ((t*)xmalloc((n)*sizeof(t)))
#define TREALLOC(s,n,t) (s=((t*)xrealloc(s,(n)*sizeof(t))))

#define EXPAND_FAIL_STACK() DOUBLE_STACK(unsigned char*)
#define ENSURE_FAIL_STACK(n)                                            \
  do {                                                                  \
    if (stacke - stackp <= (n)) {                                       \
        /* if (len > re_max_failures * MAX_NUM_FAILURE_ITEMS)           \
           {                                                            \
           FREE_AND_RETURN(stackb,(-2));                                \
           }*/                                                          \
                                                                        \
        /* Roughly double the size of the stack.  */                    \
        EXPAND_FAIL_STACK();                                            \
      }                                                                 \
  } while (0)

/* Get the interface, including the syntax bits.  */
#include "regex.h"

/* Subroutines for re_compile_pattern.  */
static void store_jump (char*, int, char*);
static void insert_jump (int, char*, char*, char*);
static void store_jump_n (char*, int, char*, unsigned);
static void insert_jump_n (int, char*, char*, char*, unsigned);
static void insert_op (int, char*, char*);
static void insert_op_2 (int, char*, char*, int, int);
static int memcmp_translate (unsigned char*, unsigned char*, int);

/* Define the syntax stuff, so we can do the <, >, etc.  */

/* This must be nonzero for the wordchar and notwordchar pattern
   commands in re_match.  */
#define Sword  1
#define Sword2 2

#define SYNTAX(c) re_syntax_table[c]

static char re_syntax_table[256];
static void init_syntax_once (void);
static const unsigned char *translate = 0;
static void init_regs (struct re_registers*, unsigned int);
static void bm_init_skip (int *, unsigned char*, int, const unsigned char*);
static int current_mbctype = MBCTYPE_ASCII;

#undef P

#ifdef RUBY
#include "util.h"
void rb_warn (const char*, ...);
# define re_warning(x) rb_warn(x)
#endif

#ifndef re_warning
# define re_warning(x)
#endif

static void
init_syntax_once()
{
   register int c;
   static int done = 0;

   if (done)
     return;

   memset(re_syntax_table, 0, sizeof re_syntax_table);

   for (c=0; c<=0x7f; c++)
     if (isalnum(c)) 
       re_syntax_table[c] = Sword;
   re_syntax_table['_'] = Sword;

   for (c=0x80; c<=0xff; c++)
     if (isalnum(c)) 
       re_syntax_table[c] = Sword2;
   done = 1;
}

void
re_set_casetable(table)
     const char *table;
{
  translate = (const unsigned char*)table;
}

/* Jim Meyering writes:

   "... Some ctype macros are valid only for character codes that
   isascii says are ASCII (SGI's IRIX-4.0.5 is one such system --when
   using /bin/cc or gcc but without giving an ansi option).  So, all
   ctype uses should be through macros like ISPRINT...  If
   STDC_HEADERS is defined, then autoconf has verified that the ctype
   macros don't need to be guarded with references to isascii. ...
   Defining isascii to 1 should let any compiler worth its salt
   eliminate the && through constant folding."
   Solaris defines some of these symbols so we must undefine them first.  */

#undef ISASCII
#if defined STDC_HEADERS || (!defined isascii && !defined HAVE_ISASCII)
# define ISASCII(c) 1
#else
# define ISASCII(c) isascii(c)
#endif

#ifdef isblank
# define ISBLANK(c) (ISASCII(c) && isblank(c))
#else
# define ISBLANK(c) ((c) == ' ' || (c) == '\t')
#endif
#ifdef isgraph
# define ISGRAPH(c) (ISASCII(c) && isgraph(c))
#else
# define ISGRAPH(c) (ISASCII(c) && isprint(c) && !isspace(c))
#endif

#undef ISPRINT
#define ISPRINT(c) (ISASCII(c) && isprint(c))
#define ISDIGIT(c) (ISASCII(c) && isdigit(c))
#define ISALNUM(c) (ISASCII(c) && isalnum(c))
#define ISALPHA(c) (ISASCII(c) && isalpha(c))
#define ISCNTRL(c) (ISASCII(c) && iscntrl(c))
#define ISLOWER(c) (ISASCII(c) && islower(c))
#define ISPUNCT(c) (ISASCII(c) && ispunct(c))
#define ISSPACE(c) (ISASCII(c) && isspace(c))
#define ISUPPER(c) (ISASCII(c) && isupper(c))
#define ISXDIGIT(c) (ISASCII(c) && isxdigit(c))

#ifndef NULL
# define NULL (void *)0
#endif

/* We remove any previous definition of `SIGN_EXTEND_CHAR',
   since ours (we hope) works properly with all combinations of
   machines, compilers, `char' and `unsigned char' argument types.
   (Per Bothner suggested the basic approach.)  */
#undef SIGN_EXTEND_CHAR
#if __STDC__
# define SIGN_EXTEND_CHAR(c) ((signed char)(c))
#else  /* not __STDC__ */
/* As in Harbison and Steele.  */
# define SIGN_EXTEND_CHAR(c) ((((unsigned char)(c)) ^ 128) - 128)
#endif

/* These are the command codes that appear in compiled regular
   expressions, one per byte.  Some command codes are followed by
   argument bytes.  A command code can specify any interpretation
   whatsoever for its arguments.  Zero-bytes may appear in the compiled
   regular expression.

   The value of `exactn' is needed in search.c (search_buffer) in emacs.
   So regex.h defines a symbol `RE_EXACTN_VALUE' to be 1; the value of
   `exactn' we use here must also be 1.  */

enum regexpcode
  {
    unused=0,
    exactn=1, /* Followed by one byte giving n, then by n literal bytes.  */
    begline,  /* Fail unless at beginning of line.  */
    endline,  /* Fail unless at end of line.  */
    begbuf,   /* Succeeds if at beginning of buffer (if emacs) or at beginning
                 of string to be matched (if not).  */
    endbuf,   /* Analogously, for end of buffer/string.  */
    endbuf2,  /* End of buffer/string, or newline just before it.  */
    begpos,   /* Matches where last scan//gsub left off.  */
    jump,     /* Followed by two bytes giving relative address to jump to.  */
    jump_past_alt,/* Same as jump, but marks the end of an alternative.  */
    on_failure_jump,     /* Followed by two bytes giving relative address of 
                            place to resume at in case of failure.  */
    finalize_jump,       /* Throw away latest failure point and then jump to 
                            address.  */
    maybe_finalize_jump, /* Like jump but finalize if safe to do so.
                            This is used to jump back to the beginning
                            of a repeat.  If the command that follows
                            this jump is clearly incompatible with the
                            one at the beginning of the repeat, such that
                            we can be sure that there is no use backtracking
                            out of repetitions already completed,
                            then we finalize.  */
    dummy_failure_jump,  /* Jump, and push a dummy failure point. This 
                            failure point will be thrown away if an attempt 
                            is made to use it for a failure. A + construct 
                            makes this before the first repeat.  Also
                            use it as an intermediary kind of jump when
                            compiling an or construct.  */
    push_dummy_failure, /* Push a dummy failure point and continue.  Used at the end of
                           alternatives.  */
    succeed_n,   /* Used like on_failure_jump except has to succeed n times;
                    then gets turned into an on_failure_jump. The relative
                    address following it is useless until then.  The
                    address is followed by two bytes containing n.  */
    jump_n,      /* Similar to jump, but jump n times only; also the relative
                    address following is in turn followed by yet two more bytes
                    containing n.  */
    try_next,    /* Jump to next pattern for the first time,
                    leaving this pattern on the failure stack. */
    finalize_push,      /* Finalize stack and push the beginning of the pattern
                           on the stack to retry (used for non-greedy match) */
    finalize_push_n,    /* Similar to finalize_push, buf finalize n time only */
    set_number_at,      /* Set the following relative location to the
                           subsequent number.  */
    anychar,     /* Matches any (more or less) one character excluding newlines.  */
    anychar_repeat,      /* Matches sequence of characters excluding newlines.  */
    charset,     /* Matches any one char belonging to specified set.
                    First following byte is number of bitmap bytes.
                    Then come bytes for a bitmap saying which chars are in.
                    Bits in each byte are ordered low-bit-first.
                    A character is in the set if its bit is 1.
                    A character too large to have a bit in the map
                    is automatically not in the set.  */
    charset_not, /* Same parameters as charset, but match any character
                    that is not one of those specified.  */
    start_memory, /* Start remembering the text that is matched, for
                    storing in a memory register.  Followed by one
                    byte containing the register number.  Register numbers
                    must be in the range 0 through RE_NREGS.  */
    stop_memory, /* Stop remembering the text that is matched
                    and store it in a memory register.  Followed by
                    one byte containing the register number. Register
                    numbers must be in the range 0 through RE_NREGS.  */
    start_paren,    /* Place holder at the start of (?:..). */
    stop_paren,    /* Place holder at the end of (?:..). */
    casefold_on,   /* Turn on casefold flag. */
    casefold_off,  /* Turn off casefold flag. */
    option_set,    /* Turn on multi line match (match with newlines). */
    start_nowidth, /* Save string point to the stack. */
    stop_nowidth,  /* Restore string place at the point start_nowidth. */
    pop_and_fail,  /* Fail after popping nowidth entry from stack. */
    stop_backtrack,  /* Restore backtrack stack at the point start_nowidth. */
    duplicate,   /* Match a duplicate of something remembered.
                    Followed by one byte containing the index of the memory 
                    register.  */
    wordchar,    /* Matches any word-constituent character.  */
    notwordchar, /* Matches any char that is not a word-constituent.  */
    wordbeg,     /* Succeeds if at word beginning.  */
    wordend,     /* Succeeds if at word end.  */
    wordbound,   /* Succeeds if at a word boundary.  */
    notwordbound /* Succeeds if not at a word boundary.  */
  };


/* Number of failure points to allocate space for initially,
   when matching.  If this number is exceeded, more space is allocated,
   so it is not a hard limit.  */

#ifndef NFAILURES
#define NFAILURES 160
#endif

/* Store NUMBER in two contiguous bytes starting at DESTINATION.  */
#define STORE_NUMBER(destination, number)                               \
  do { (destination)[0] = (number) & 0377;                              \
    (destination)[1] = (number) >> 8; } while (0)

/* Same as STORE_NUMBER, except increment the destination pointer to
   the byte after where the number is stored.  Watch out that values for
   DESTINATION such as p + 1 won't work, whereas p will.  */
#define STORE_NUMBER_AND_INCR(destination, number)                      \
  do { STORE_NUMBER(destination, number);                               \
    (destination) += 2; } while (0)


/* Put into DESTINATION a number stored in two contingous bytes starting
   at SOURCE.  */
#define EXTRACT_NUMBER(destination, source)                             \
  do { (destination) = *(source) & 0377;                                \
    (destination) += SIGN_EXTEND_CHAR(*(char*)((source) + 1)) << 8; } while (0)

/* Same as EXTRACT_NUMBER, except increment the pointer for source to
   point to second byte of SOURCE.  Note that SOURCE has to be a value
   such as p, not, e.g., p + 1. */
#define EXTRACT_NUMBER_AND_INCR(destination, source)                    \
  do { EXTRACT_NUMBER(destination, source);                             \
       (source) += 2; } while (0)


/* Specify the precise syntax of regexps for compilation.  This provides
   for compatibility for various utilities which historically have
   different, incompatible syntaxes.

   The argument SYNTAX is a bit-mask comprised of the various bits
   defined in regex.h.  */

long
re_set_syntax(syntax)
  long syntax;
{
    /* obsolete */
    return 0;
}

/* Macros for re_compile_pattern, which is found below these definitions.  */

#define TRANSLATE_P() ((options&RE_OPTION_IGNORECASE) && translate)
#define MAY_TRANSLATE() ((bufp->options&(RE_OPTION_IGNORECASE|RE_MAY_IGNORECASE)) && translate)
/* Fetch the next character in the uncompiled pattern---translating it 
   if necessary.  Also cast from a signed character in the constant
   string passed to us by the user to an unsigned char that we can use
   as an array index (in, e.g., `translate').  */
#define PATFETCH(c)                                                     \
  do {if (p == pend) goto end_of_pattern;                               \
    c = (unsigned char) *p++;                                           \
    if (TRANSLATE_P()) c = (unsigned char)translate[c]; \
  } while (0)

/* Fetch the next character in the uncompiled pattern, with no
   translation.  */
#define PATFETCH_RAW(c)                                                 \
  do {if (p == pend) goto end_of_pattern;                               \
    c = (unsigned char)*p++;                                            \
  } while (0)

/* Go backwards one character in the pattern.  */
#define PATUNFETCH p--

#define MBC2WC(c, p)                                                    \
  do {                                                                  \
    if (current_mbctype == MBCTYPE_UTF8) {                              \
      int n = mbclen(c) - 1;                                            \
      c &= (1<<(BYTEWIDTH-2-n)) - 1;                                    \
      while (n--) {                                                     \
        c = c << 6 | (*p++ & ((1<<6)-1));                               \
      }                                                                 \
    }                                                                   \
    else {                                                              \
      c <<= 8;                                                          \
      c |= (unsigned char)*(p)++;                                       \
    }                                                                   \
  } while (0)

#define PATFETCH_MBC(c)                                                 \
  do {                                                                  \
    if (p + mbclen(c) - 1 >= pend) goto end_of_pattern;                 \
    MBC2WC(c, p);                                                       \
  } while(0)

#define WC2MBC1ST(c)                                                    \
 ((current_mbctype != MBCTYPE_UTF8) ? ((c<0x100) ? (c) : (((c)>>8)&0xff)) : utf8_firstbyte(c))

typedef unsigned int (*mbc_startpos_func_t) (const char *string, unsigned int pos);

static unsigned int asc_startpos (const char *string, unsigned int pos);
static unsigned int euc_startpos (const char *string, unsigned int pos);
static unsigned int sjis_startpos (const char *string, unsigned int pos);
static unsigned int utf8_startpos (const char *string, unsigned int pos);

static const mbc_startpos_func_t mbc_startpos_func[4] = {
  asc_startpos, euc_startpos, sjis_startpos, utf8_startpos
};

#define mbc_startpos(start, pos) (*mbc_startpos_func[current_mbctype])((start), (pos))

static unsigned int
utf8_firstbyte(c)
     unsigned long c;
{
  if (c < 0x80) return c;
  if (c <= 0x7ff) return ((c>>6)&0xff)|0xc0;
  if (c <= 0xffff) return ((c>>12)&0xff)|0xe0;
  if (c <= 0x1fffff) return ((c>>18)&0xff)|0xf0;
  if (c <= 0x3ffffff) return ((c>>24)&0xff)|0xf8;
  if (c <= 0x7fffffff) return ((c>>30)&0xff)|0xfc;
#if SIZEOF_INT > 4
  if (c <= 0xfffffffff) return 0xfe;
#else
  return 0xfe;
#endif
}

static void
print_mbc(c)
     unsigned int c;
{
  if (current_mbctype == MBCTYPE_UTF8) {
    if (c < 0x80)
      printf("%c", (int)c);
    else if (c <= 0x7ff)
      printf("%c%c", (int)utf8_firstbyte(c), (int)(c & 0x3f));
    else if (c <= 0xffff)
      printf("%c%c%c", (int)utf8_firstbyte(c), (int)((c >> 6) & 0x3f),
             (int)(c & 0x3f));
    else if (c <= 0x1fffff) 
      printf("%c%c%c%c", (int)utf8_firstbyte(c), (int)((c >> 12) & 0x3f),
             (int)((c >> 6) & 0x3f), (int)(c & 0x3f));
    else if (c <= 0x3ffffff)
      printf("%c%c%c%c%c", (int)utf8_firstbyte(c), (int)((c >> 18) & 0x3f),
             (int)((c >> 12) & 0x3f), (int)((c >> 6) & 0x3f), (int)(c & 0x3f));
    else if (c <= 0x7fffffff)
      printf("%c%c%c%c%c%c", (int)utf8_firstbyte(c), (int)((c >> 24) & 0x3f),
             (int)((c >> 18) & 0x3f), (int)((c >> 12) & 0x3f),
             (int)((c >> 6) & 0x3f), (int)(c & 0x3f));
  }
  else if (c < 0xff) {
    printf("\\%o", (int)c);
  }
  else {
    printf("%c%c", (int)(c >> BYTEWIDTH), (int)(c &0xff));
  }
}

/* If the buffer isn't allocated when it comes in, use this.  */
#define INIT_BUF_SIZE  28

/* Make sure we have at least N more bytes of space in buffer.  */
#define GET_BUFFER_SPACE(n)                                             \
  do {                                                                  \
    while (b - bufp->buffer + (n) >= bufp->allocated)                   \
      EXTEND_BUFFER;                                                    \
  } while (0)

/* Make sure we have one more byte of buffer space and then add CH to it.  */
#define BUFPUSH(ch)                                                     \
  do {                                                                  \
    GET_BUFFER_SPACE(1);                                                \
    *b++ = (char)(ch);                                                  \
  } while (0)

/* Extend the buffer by twice its current size via reallociation and
   reset the pointers that pointed into the old allocation to point to
   the correct places in the new allocation.  If extending the buffer
   results in it being larger than 1 << 16, then flag memory exhausted.  */
#define EXTEND_BUFFER                                           \
  do { char *old_buffer = bufp->buffer;                                 \
    if (bufp->allocated == (1L<<16)) goto too_big;                      \
    bufp->allocated *= 2;                                               \
    if (bufp->allocated > (1L<<16)) bufp->allocated = (1L<<16);         \
    bufp->buffer = (char*)xrealloc(bufp->buffer, bufp->allocated);      \
    if (bufp->buffer == 0)                                              \
      goto memory_exhausted;                                            \
    b = (b - old_buffer) + bufp->buffer;                                \
    if (fixup_alt_jump)                                                 \
      fixup_alt_jump = (fixup_alt_jump - old_buffer) + bufp->buffer;    \
    if (laststart)                                                      \
      laststart = (laststart - old_buffer) + bufp->buffer;              \
    begalt = (begalt - old_buffer) + bufp->buffer;                      \
    if (pending_exact)                                                  \
      pending_exact = (pending_exact - old_buffer) + bufp->buffer;      \
  } while (0)


/* Set the bit for character C in a character set list.  */
#define SET_LIST_BIT(c)                                                 \
  (b[(unsigned char)(c) / BYTEWIDTH]                                    \
   |= 1 << ((unsigned char)(c) % BYTEWIDTH))

/* Get the next unsigned number in the uncompiled pattern.  */
#define GET_UNSIGNED_NUMBER(num)                                        \
  do { if (p != pend) {                                                 \
        PATFETCH(c);                                                    \
        while (ISDIGIT(c)) {                                            \
          if (num < 0)                                                  \
             num = 0;                                                   \
          num = num * 10 + c - '0';                                     \
          if (p == pend)                                                \
             break;                                                     \
          PATFETCH(c);                                                  \
        }                                                               \
     }                                                                  \
  } while (0)

#define STREQ(s1, s2) ((strcmp(s1, s2) == 0))

#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */

#define IS_CHAR_CLASS(string)                                           \
   (STREQ(string, "alpha") || STREQ(string, "upper")                    \
    || STREQ(string, "lower") || STREQ(string, "digit")                 \
    || STREQ(string, "alnum") || STREQ(string, "xdigit")                \
    || STREQ(string, "space") || STREQ(string, "print")                 \
    || STREQ(string, "punct") || STREQ(string, "graph")                 \
    || STREQ(string, "cntrl") || STREQ(string, "blank"))

#define STORE_MBC(p, c)                                                 \
  do {                                                                  \
    (p)[0] = (unsigned char)(((c) >>24) & 0xff);                        \
    (p)[1] = (unsigned char)(((c) >>16) & 0xff);                        \
    (p)[2] = (unsigned char)(((c) >> 8) & 0xff);                        \
    (p)[3] = (unsigned char)(((c) >> 0) & 0xff);                        \
  } while (0)

#define STORE_MBC_AND_INCR(p, c)                                        \
  do {                                                                  \
    *(p)++ = (unsigned char)(((c) >>24) & 0xff);                        \
    *(p)++ = (unsigned char)(((c) >>16) & 0xff);                        \
    *(p)++ = (unsigned char)(((c) >> 8) & 0xff);                        \
    *(p)++ = (unsigned char)(((c) >> 0) & 0xff);                        \
  } while (0)

#define EXTRACT_MBC(p)                                                  \
  ((unsigned int)((unsigned char)(p)[0] << 24 |                         \
                    (unsigned char)(p)[1] << 16 |                       \
                    (unsigned char)(p)[2] <<  8 |                       \
                    (unsigned char)(p)[3]))

#define EXTRACT_MBC_AND_INCR(p)                                         \
  ((unsigned int)((p) += 4,                                             \
                    (unsigned char)(p)[-4] << 24 |                      \
                    (unsigned char)(p)[-3] << 16 |                      \
                    (unsigned char)(p)[-2] <<  8 |                      \
                    (unsigned char)(p)[-1]))

#define EXTRACT_UNSIGNED(p) \
  ((unsigned char)(p)[0] | (unsigned char)(p)[1] << 8)
#define EXTRACT_UNSIGNED_AND_INCR(p) \
  ((p) += 2, (unsigned char)(p)[-2] | (unsigned char)(p)[-1] << 8)

/* Handle (mb)?charset(_not)?.

   Structure of mbcharset(_not)? in compiled pattern.

     struct {
       unsinged char id;                mbcharset(_not)?
       unsigned char sbc_size;
       unsigned char sbc_map[sbc_size]; same as charset(_not)? up to here.
       unsigned short mbc_size;         number of intervals.
       struct {
         unsigned long beg;             beginning of interval.
         unsigned long end;             end of interval.
       } intervals[mbc_size];
     }; */

static void
set_list_bits(c1, c2, b)
    unsigned long c1, c2;
    unsigned char *b;
{
  unsigned char sbc_size = b[-1];
  unsigned short mbc_size = EXTRACT_UNSIGNED(&b[sbc_size]);
  unsigned short beg, end, upb;

  if (c1 > c2)
    return;
  b = &b[sbc_size + 2];

  for (beg = 0, upb = mbc_size; beg < upb; ) {
    unsigned short mid = (unsigned short)(beg + upb) >> 1;

    if ((int)c1 - 1 > (int)EXTRACT_MBC(&b[mid*8+4]))
      beg = mid + 1;
    else
      upb = mid;
  }

  for (end = beg, upb = mbc_size; end < upb; ) {
    unsigned short mid = (unsigned short)(end + upb) >> 1;

    if ((int)c2 >= (int)EXTRACT_MBC(&b[mid*8]) - 1)
      end = mid + 1;
    else
      upb = mid;
  }

  if (beg != end) {
    if (c1 > EXTRACT_MBC(&b[beg*8]))
      c1 = EXTRACT_MBC(&b[beg*8]);
    if (c2 < EXTRACT_MBC(&b[(end - 1)*8+4]))
      c2 = EXTRACT_MBC(&b[(end - 1)*8+4]);
  }
  if (end < mbc_size && end != beg + 1)
    /* NOTE: memcpy() would not work here.  */
    memmove(&b[(beg + 1)*8], &b[end*8], (mbc_size - end)*8);
  STORE_MBC(&b[beg*8 + 0], c1);
  STORE_MBC(&b[beg*8 + 4], c2);
  mbc_size += beg - end + 1;
  STORE_NUMBER(&b[-2], mbc_size);
}

static int
is_in_list_sbc(c, b)
    unsigned long c;
    const unsigned char *b;
{
  unsigned short size;

  size = *b++;
  return ((int)c / BYTEWIDTH < (int)size && b[c / BYTEWIDTH] & 1 << c % BYTEWIDTH);
}
  
static int
is_in_list_mbc(c, b)
    unsigned long c;
    const unsigned char *b;
{
  unsigned short size;
  unsigned short i, j;

  size = *b++;
  b += size + 2;
  size = EXTRACT_UNSIGNED(&b[-2]);
  if (size == 0) return 0;

  for (i = 0, j = size; i < j; ) {
    unsigned short k = (unsigned short)(i + j) >> 1;

    if (c > EXTRACT_MBC(&b[k*8+4]))
      i = k + 1;
    else
      j = k;
  }
  if (i < size && EXTRACT_MBC(&b[i*8]) <= c)
    return 1;

  return 0;
}

static int
is_in_list(c, b)
    unsigned long c;
    const unsigned char *b;
{
  return is_in_list_sbc(c, b) || (current_mbctype ? is_in_list_mbc(c, b) : 0);
}

static void
print_partial_compiled_pattern(start, end)
    unsigned char *start;
    unsigned char *end;
{
  int mcnt, mcnt2;
  unsigned char *p = start;
  unsigned char *pend = end;

  if (start == NULL) {
    printf("(null)\n");
    return;
  }

  /* Loop over pattern commands.  */
  while (p < pend) {
    switch ((enum regexpcode)*p++) {
    case unused:
      printf("/unused");
      break;

    case exactn:
      mcnt = *p++;
      printf("/exactn/%d", mcnt);
      do {
        putchar('/');
        printf("%c", *p++);
      }
      while (--mcnt);
      break;

    case start_memory:
      mcnt = *p++;
      printf("/start_memory/%d/%d", mcnt, *p++);
      break;

    case stop_memory:
      mcnt = *p++;
      printf("/stop_memory/%d/%d", mcnt, *p++);
      break;

    case start_paren:
      printf("/start_paren");
      break;

    case stop_paren:
      printf("/stop_paren");
      break;

    case casefold_on:
      printf("/casefold_on");
      break;

    case casefold_off:
      printf("/casefold_off");
      break;

    case option_set:
      printf("/option_set/%d", *p++);
      break;

    case start_nowidth:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/start_nowidth//%d", mcnt);
      break;

    case stop_nowidth:
      printf("/stop_nowidth//");
      p += 2;
      break;

    case pop_and_fail:
      printf("/pop_and_fail");
      break;

    case stop_backtrack:
      printf("/stop_backtrack//");
      p += 2;
      break;

    case duplicate:
      printf("/duplicate/%d", *p++);
      break;

    case anychar:
      printf("/anychar");
      break;

    case anychar_repeat:
      printf("/anychar_repeat");
      break;

    case charset:
    case charset_not:
      {
        register int c;

        printf("/charset%s",
               (enum regexpcode)*(p - 1) == charset_not ? "_not" : "");

        mcnt = *p++;
        printf("/%d", mcnt);
        for (c = 0; c < mcnt; c++) {
          unsigned bit;
          unsigned char map_byte = p[c];

          putchar('/');

          for (bit = 0; bit < BYTEWIDTH; bit++)
            if (map_byte & (1 << bit))
              printf("%c", c * BYTEWIDTH + bit);
        }
        p += mcnt;
        mcnt = EXTRACT_UNSIGNED_AND_INCR(p);
        putchar('/');
        while (mcnt--) {
          print_mbc(EXTRACT_MBC_AND_INCR(p));
          putchar('-');
          print_mbc(EXTRACT_MBC_AND_INCR(p));
        }
        break;
      }

    case begline:
      printf("/begline");
      break;

    case endline:
      printf("/endline");
      break;

    case on_failure_jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/on_failure_jump//%d", mcnt);
      break;

    case dummy_failure_jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/dummy_failure_jump//%d", mcnt);
      break;

    case push_dummy_failure:
      printf("/push_dummy_failure");
      break;

    case finalize_jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/finalize_jump//%d", mcnt);
      break;

    case maybe_finalize_jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/maybe_finalize_jump//%d", mcnt);
      break;

    case jump_past_alt:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/jump_past_alt//%d", mcnt);
      break;

    case jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/jump//%d", mcnt);
      break;

    case succeed_n: 
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      EXTRACT_NUMBER_AND_INCR(mcnt2, p);
      printf("/succeed_n//%d//%d", mcnt, mcnt2);
      break;

    case jump_n: 
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      EXTRACT_NUMBER_AND_INCR(mcnt2, p);
      printf("/jump_n//%d//%d", mcnt, mcnt2);
      break;

    case set_number_at: 
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      EXTRACT_NUMBER_AND_INCR(mcnt2, p);
      printf("/set_number_at//%d//%d", mcnt, mcnt2);
      break;

    case try_next:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/try_next//%d", mcnt);
      break;

    case finalize_push:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      printf("/finalize_push//%d", mcnt);
      break;

    case finalize_push_n:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      EXTRACT_NUMBER_AND_INCR(mcnt2, p);
      printf("/finalize_push_n//%d//%d", mcnt, mcnt2);
      break;

    case wordbound:
      printf("/wordbound");
      break;

    case notwordbound:
      printf("/notwordbound");
      break;

    case wordbeg:
      printf("/wordbeg");
      break;

    case wordend:
      printf("/wordend");

    case wordchar:
      printf("/wordchar");
      break;
          
    case notwordchar:
      printf("/notwordchar");
      break;

    case begbuf:
      printf("/begbuf");
      break;

    case endbuf:
      printf("/endbuf");
      break;

    case endbuf2:
      printf("/endbuf2");
      break;

    case begpos:
      printf("/begpos");
      break;

    default:
      printf("?%d", *(p-1));
    }
  }
  printf("/\n");
}


static void
print_compiled_pattern(bufp)
     struct re_pattern_buffer *bufp;
{
  unsigned char *buffer = (unsigned char*)bufp->buffer;

  print_partial_compiled_pattern(buffer, buffer + bufp->used);
}

static char*
calculate_must_string(start, end)
     char *start;
     char *end;
{
  int mcnt;
  int max = 0;
  unsigned char *p = start;
  unsigned char *pend = end;
  char *must = 0;

  if (start == NULL) return 0;

  /* Loop over pattern commands.  */
  while (p < pend) {
    switch ((enum regexpcode)*p++) {
    case unused:
      break;

    case exactn:
      mcnt = *p;
      if (mcnt > max) {
        must = p;
        max = mcnt;
      }
      p += mcnt+1;
      break;

    case start_memory:
    case stop_memory:
      p += 2;
      break;

    case duplicate:
    case option_set:
      p++;
      break;

    case casefold_on:
    case casefold_off:
      return 0;         /* should not check must_string */

    case pop_and_fail:
    case anychar:
    case anychar_repeat:
    case begline:
    case endline:
    case wordbound:
    case notwordbound:
    case wordbeg:
    case wordend:
    case wordchar:
    case notwordchar:
    case begbuf:
    case endbuf:
    case endbuf2:
    case begpos:
    case push_dummy_failure:
    case start_paren:
    case stop_paren:
      break;

    case charset:
    case charset_not:
      mcnt = *p++;
      p += mcnt;
      mcnt = EXTRACT_UNSIGNED_AND_INCR(p);
      while (mcnt--) {
        p += 8;
      }
      break;

    case on_failure_jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      if (mcnt > 0) p += mcnt;
      if ((enum regexpcode)p[-3] == jump) {
        p -= 2;
        EXTRACT_NUMBER_AND_INCR(mcnt, p);
        if (mcnt > 0) p += mcnt;
      }
      break;

    case dummy_failure_jump:
    case succeed_n: 
    case try_next:
    case jump:
      EXTRACT_NUMBER_AND_INCR(mcnt, p);
      if (mcnt > 0) p += mcnt;
      break;

    case start_nowidth:
    case stop_nowidth:
    case stop_backtrack:
    case finalize_jump:
    case maybe_finalize_jump:
    case finalize_push:
      p += 2;
      break;

    case jump_n: 
    case set_number_at: 
    case finalize_push_n:
      p += 4;
      break;

    default:
      break;
    }
  }
  return must;
}

static unsigned int
read_backslash(c)
     int c;
{
  switch (c) {
  case 'n':
    return '\n';

  case 't':
    return '\t';

  case 'r':
    return '\r';

  case 'f':
    return '\f';

  case 'v':
    return '\v';

  case 'a':
    return '\007';

  case 'b':
    return '\010';

  case 'e':
    return '\033';
  }
  return c;
}

static unsigned int
read_special(p, pend, pp)
     const char *p, *pend, **pp;
{
  int c;

  PATFETCH_RAW(c);
  switch (c) {
  case 'M':
    PATFETCH_RAW(c);
    if (c != '-') return -1;
    PATFETCH_RAW(c);
    *pp = p;
    if (c == '\\') {
      return read_special(--p, pend, pp) | 0x80;
    }
    else if (c == -1) return ~0;
    else {
      return ((c & 0xff) | 0x80);
    }

  case 'C':
    PATFETCH_RAW(c);
    if (c != '-') return ~0;
  case 'c':
    PATFETCH_RAW(c);
    *pp = p;
    if (c == '\\') {
      c = read_special(--p, pend, pp);
    }
    else if (c == '?') return 0177;
    else if (c == -1) return ~0;
    return c & 0x9f;
  default:
    *pp = p + 1;
    return read_backslash(c);
  }

 end_of_pattern:
  return ~0;
}

/* re_compile_pattern takes a regular-expression string
   and converts it into a buffer full of byte commands for matching.

   PATTERN   is the address of the pattern string
   SIZE      is the length of it.
   BUFP     is a  struct re_pattern_buffer *  which points to the info
             on where to store the byte commands.
             This structure contains a  char *  which points to the
             actual space, which should have been obtained with malloc.
             re_compile_pattern may use realloc to grow the buffer space.

   The number of bytes of commands can be found out by looking in
   the `struct re_pattern_buffer' that bufp pointed to, after
   re_compile_pattern returns. */

char *
re_compile_pattern(pattern, size, bufp)
     const char *pattern;
     int size;
     struct re_pattern_buffer *bufp;
{
  register char *b = bufp->buffer;
  register const char *p = pattern;
  const char *nextp;
  const char *pend = pattern + size;
  register unsigned int c, c1 = 0;
  const char *p0;
  int numlen;
#define ERROR_MSG_MAX_SIZE 200
  static char error_msg[ERROR_MSG_MAX_SIZE+1];

  /* Address of the count-byte of the most recently inserted `exactn'
     command.  This makes it possible to tell whether a new exact-match
     character can be added to that command or requires a new `exactn'
     command.  */

  char *pending_exact = 0;

  /* Address of the place where a forward-jump should go to the end of
     the containing expression.  Each alternative of an `or', except the
     last, ends with a forward-jump of this sort.  */

  char *fixup_alt_jump = 0;

  /* Address of start of the most recently finished expression.
     This tells postfix * where to find the start of its operand.  */

  char *laststart = 0;

  /* In processing a repeat, 1 means zero matches is allowed.  */

  char zero_times_ok;

  /* In processing a repeat, 1 means many matches is allowed.  */

  char many_times_ok;

  /* In processing a repeat, 1 means non-greedy matches.  */

  char greedy;

  /* Address of beginning of regexp, or inside of last (.  */

  char *begalt = b;

  /* Place in the uncompiled pattern (i.e., the {) to
     which to go back if the interval is invalid.  */
  const char *beg_interval;

  /* In processing an interval, at least this many matches must be made.  */
  int lower_bound;

  /* In processing an interval, at most this many matches can be made.  */
  int upper_bound;

  /* Stack of information saved by ( and restored by ).
     Five stack elements are pushed by each (:
     First, the value of b.
     Second, the value of fixup_alt_jump.
     Third, the value of begalt.
     Fourth, the value of regnum.
     Fifth, the type of the paren. */

  int stacka[40];
  int *stackb = stacka;
  int *stackp = stackb;
  int *stacke = stackb + 40;

  /* Counts ('s as they are encountered.  Remembered for the matching ),
     where it becomes the register number to put in the stop_memory
     command.  */

  int regnum = 1;

  int range = 0;
  int had_mbchar = 0;
  int had_num_literal = 0;
  int had_char_class = 0;

  int options = bufp->options;

  bufp->fastmap_accurate = 0;
  bufp->must = 0;
  bufp->must_skip = 0;

  /* Initialize the syntax table.  */
  init_syntax_once();

  if (bufp->allocated == 0) {
    bufp->allocated = INIT_BUF_SIZE;
    /* EXTEND_BUFFER loses when bufp->allocated is 0.  */
    bufp->buffer = (char*)xrealloc(bufp->buffer, INIT_BUF_SIZE);
    if (!bufp->buffer) goto memory_exhausted; /* this not happen */
    begalt = b = bufp->buffer;
  }

  while (p != pend) {
    PATFETCH(c);

    switch (c) {
    case '$':
      if (bufp->options & RE_OPTION_SINGLELINE) {
        BUFPUSH(endbuf);
      }
      else {
        p0 = p;
        /* When testing what follows the $,
           look past the \-constructs that don't consume anything.  */

        while (p0 != pend) {
          if (*p0 == '\\' && p0 + 1 != pend
              && (p0[1] == 'b' || p0[1] == 'B'))
            p0 += 2;
          else
            break;
        }
        BUFPUSH(endline);
      }
      break;

    case '^':
      if (bufp->options & RE_OPTION_SINGLELINE)
        BUFPUSH(begbuf);
      else
        BUFPUSH(begline);
      break;

    case '+':
    case '?':
    case '*':
      /* If there is no previous pattern, char not special. */
      if (!laststart) {
        snprintf(error_msg, ERROR_MSG_MAX_SIZE, 
                 "invalid regular expression; there's no previous pattern, to which '%c' would define cardinality at %d", 
                 c, p-pattern);
        FREE_AND_RETURN(stackb, error_msg);
      }
      /* If there is a sequence of repetition chars,
         collapse it down to just one.  */
      zero_times_ok = c != '+';
      many_times_ok = c != '?';
      greedy = 1;
      if (p != pend) {
        PATFETCH(c);
        switch (c) {
        case '?':
          greedy = 0;
          break;
        case '*':
        case '+':
          goto nested_meta;
        default:
          PATUNFETCH;
          break;
        }
      }

    repeat:
      /* Star, etc. applied to an empty pattern is equivalent
         to an empty pattern.  */
      if (!laststart)  
        break;

      if (greedy && many_times_ok && *laststart == anychar && b - laststart <= 2) {
        if (b[-1] == stop_paren)
          b--;
        if (zero_times_ok)
          *laststart = anychar_repeat;
        else {
          BUFPUSH(anychar_repeat);
        }
        break;
      }
      /* Now we know whether or not zero matches is allowed
         and also whether or not two or more matches is allowed.  */
      if (many_times_ok) {
        /* If more than one repetition is allowed, put in at the
           end a backward relative jump from b to before the next
           jump we're going to put in below (which jumps from
           laststart to after this jump).  */
        GET_BUFFER_SPACE(3);
        store_jump(b,greedy?maybe_finalize_jump:finalize_push,laststart-3);
        b += 3;         /* Because store_jump put stuff here.  */
      }

      /* On failure, jump from laststart to next pattern, which will be the
         end of the buffer after this jump is inserted.  */
      GET_BUFFER_SPACE(3);
      insert_jump(on_failure_jump, laststart, b + 3, b);
      b += 3;

      if (zero_times_ok) {
        if (greedy == 0) {
          GET_BUFFER_SPACE(3);
          insert_jump(try_next, laststart, b + 3, b);
          b += 3;
        }
      }
      else {
        /* At least one repetition is required, so insert a
           `dummy_failure_jump' before the initial
           `on_failure_jump' instruction of the loop. This
           effects a skip over that instruction the first time
           we hit that loop.  */
        GET_BUFFER_SPACE(3);
        insert_jump(dummy_failure_jump, laststart, laststart + 6, b);
        b += 3;
      }
      break;

    case '.':
      laststart = b;
      BUFPUSH(anychar);
      break;

    case '[':
      if (p == pend)
        FREE_AND_RETURN(stackb, "invalid regular expression; '[' can't be the last character ie. can't start range at the end of pattern");
      while ((b - bufp->buffer + 9 + (1 << BYTEWIDTH) / BYTEWIDTH)
             > bufp->allocated)
        EXTEND_BUFFER;

      laststart = b;
      if (*p == '^') {
        BUFPUSH(charset_not); 
        p++;
      }
      else
        BUFPUSH(charset);
      p0 = p;

      BUFPUSH((1 << BYTEWIDTH) / BYTEWIDTH);
      /* Clear the whole map */
      memset(b, 0, (1 << BYTEWIDTH) / BYTEWIDTH + 2);

      had_mbchar = 0;
      had_num_literal = 0;
      had_char_class = 0;

      /* Read in characters and ranges, setting map bits.  */
      for (;;) {
        int size;
        unsigned last = (unsigned)-1;

        if ((size = EXTRACT_UNSIGNED(&b[(1 << BYTEWIDTH) / BYTEWIDTH])) || current_mbctype) {
          /* Ensure the space is enough to hold another interval
             of multi-byte chars in charset(_not)?.  */
          size = (1 << BYTEWIDTH) / BYTEWIDTH + 2 + size*8 + 8;
          while (b + size + 1 > bufp->buffer + bufp->allocated)
            EXTEND_BUFFER;
        }
      range_retry:
        if (range && had_char_class) {
          FREE_AND_RETURN(stackb, "invalid regular expression; can't use character class as an end value of range");
        }
        PATFETCH_RAW(c);

        if (c == ']') {
          if (p == p0 + 1) {
            if (p == pend)
              FREE_AND_RETURN(stackb, "invalid regular expression; empty character class");
            re_warning("character class has `]' without escape");
          }
          else 
            /* Stop if this isn't merely a ] inside a bracket
               expression, but rather the end of a bracket
               expression.  */
            break;
        }
        /* Look ahead to see if it's a range when the last thing
           was a character class.  */
        if (had_char_class && c == '-' && *p != ']')
          FREE_AND_RETURN(stackb, "invalid regular expression; can't use character class as a start value of range");
        if (ismbchar(c)) {
          PATFETCH_MBC(c);
          had_mbchar++;
        }
        had_char_class = 0;

        if (c == '-' && ((p != p0 + 1 && *p != ']') ||
                         (p[0] == '-' && p[1] != ']') ||
                         range))
          re_warning("character class has `-' without escape");
        if (c == '[' && *p != ':')
          re_warning("character class has `[' without escape");

        /* \ escapes characters when inside [...].  */
        if (c == '\\') {
          PATFETCH_RAW(c);
          switch (c) {
          case 'w':
            for (c = 0; c < (1 << BYTEWIDTH); c++) {
              if (SYNTAX(c) == Sword ||
                  (!current_mbctype && SYNTAX(c) == Sword2))
                SET_LIST_BIT(c);
            }
            if (current_mbctype) {
              set_list_bits(0x80, 0xffffffff, b);
            }
            had_char_class = 1;
            last = -1;
            continue;

          case 'W':
            for (c = 0; c < (1 << BYTEWIDTH); c++) {
              if (SYNTAX(c) != Sword &&
                  ((current_mbctype && !re_mbctab[c]) ||
                  (!current_mbctype && SYNTAX(c) != Sword2)))
                SET_LIST_BIT(c);
            }
            had_char_class = 1;
            last = -1;
            continue;

          case 's':
            for (c = 0; c < 256; c++)
              if (ISSPACE(c))
                SET_LIST_BIT(c);
            had_char_class = 1;
            last = -1;
            continue;

          case 'S':
            for (c = 0; c < 256; c++)
              if (!ISSPACE(c))