/* Header for encoding conversion functions; coding-system object.
   #### rename me to coding-system.h
   Copyright (C) 1991, 1995 Free Software Foundation, Inc.
   Copyright (C) 1995 Sun Microsystems, Inc.
   Copyright (C) 2000, 2001, 2002, 2005, 2010 Ben Wing.

This file is part of XEmacs.

XEmacs is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.

XEmacs is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with XEmacs.  If not, see <http://www.gnu.org/licenses/>. */

/* Synched up with: Mule 2.3.  Not in FSF. */

/* Authorship:

   Current primary author: Ben Wing <ben@xemacs.org>

   Written by Ben Wing <ben@xemacs.org> for XEmacs, 1995, loosely based
     on code written 91.10.09 by K.Handa <handa@etl.go.jp>.
   Rewritten again 2000-2001 by Ben Wing to support properly
   abstracted coding systems.
   September 2001: Finished last part of abstraction, the detection
   mechanism.
*/

#ifndef INCLUDED_file_coding_h_
#define INCLUDED_file_coding_h_

/* Capsule description of the different structures, what their purpose is,
   how they fit together, and where various bits of data are stored.

   A "coding system" is an algorithm for converting stream data in one format
   into stream data in another format.  Currently most of the coding systems
   we have created concern internationalized text, and convert between the
   XEmacs internal format for multilingual text, and various external
   representations of such text.  However, any such conversion is possible,
   for example, compressing or uncompressing text using the gzip algorithm.
   All coding systems provide both encode and decode routines, so that the
   conversion can go both ways.  Unfortunately encoding and decoding may not
   be exact inverses, even for a specific instance of a coding system.  Care
   must be taken when this is not the case.

   The way we handle this is by dividing the various potential coding
   systems into types, analogous to classes in C++.  Each coding system
   type encompasses a series of related coding systems that it can
   implement, and it has properties which control how exactly the encoding
   works.  A particular set of values for each of the properties makes up a
   "coding system", and specifies one particular encoding.  A `struct
   Lisp_Coding_System' object encapsulates those settings -- its type, the
   values chosen for all properties of that type, a name for the coding
   system, some documentation.

   In addition, there are of course methods associated with a coding system
   type, implementing the encoding, decoding, etc.  These are stored in a
   `struct coding_system_methods' object, one per coding-system type, which
   contains mostly function pointers.  This is retrievable from the
   coding-system object (i.e. the struct Lisp_Coding_System), which has a
   pointer to it.

   In order to actually use a coding system to do an encoding or decoding
   operation, you need to use a coding Lstream.

   Now let's look more at attached data.  All coding systems have certain
   common data fields -- name, type, documentation, etc. -- as well as a
   bunch more that are defined by the coding system type.  To handle this
   cleanly, each coding system type defines a structure that holds just the
   fields of data particular to it, and calls it e.g. `struct
   iso2022_coding_system' for coding system type `iso2022'.  When the
   memory block holding the coding system object is created, it is sized
   such that it can hold both the struct Lisp_Coding_System and the struct
   iso2022_coding_system (or whatever) directly following it. (This is a
   common trick; another possibility is to have a void * pointer in the
   struct Lisp_Coding_System, which points to another memory block holding
   the struct iso2022_coding_system.) A macro is provided
   (CODING_SYSTEM_TYPE_DATA) to retrieve a pointer of the right type to the
   type-specific data contained within the overall `struct
   Lisp_Coding_System' block.

   Lstreams, similarly, are objects of type `struct lstream' holding data
   about the stream operation (how much data has been read or written, any
   buffered data, any error conditions, etc.), and like coding systems have
   different types.  They have a structure called `Lstream_implementation',
   one per lstream type, exactly analogous to `struct
   coding_system_methods'.  In addition, they have type-specific data
   (specifying, e.g., the file number, FILE *, memory location, other
   lstream, etc. to read the data from or write it to, and for conversion
   processes, the current state of the process -- are we decoding ASCII or
   Kanji characters?  are we in the middle of a processing an escape
   sequence?  etc.).  This type-specific data is stored in a structure
   named `struct coding_stream'.  Just like for coding systems, the
   type-independent data in the `struct lstream' and the type-dependent
   data in the `struct coding_stream' are stored together in the same
   memory block.

   Now things get a bit tricky.  The `struct coding_stream' is
   type-specific from the point of view of an lstream, but not from the
   point of view of a coding system.  It contains only general data about
   the conversion process, e.g. the name of the coding system used for
   conversion, the lstream that we take data from or write it to (depending
   on whether this was created as a read stream or a write stream), a
   buffer to hold extra data we retrieved but can't send on yet, some
   flags, etc.  It also needs some data specific to the particular coding
   system and thus to the particular operation going on.  This data is held
   in a structure named (e.g.) `struct iso2022_coding_stream', and it's
   held in a separate memory block and pointed to by the generic `struct
   coding_stream'.  It's not glommed into a single memory block both
   because that would require making changes to the generic lstream code
   and more importantly because the coding system used in a particular
   coding lstream can be changed at any point during the lifetime of the
   lstream, and possibly multiple times. (For example, it can be set using
   the Lisp primitives `set-process-input-coding-system' and
   `set-console-tty-input-coding-system', as well as getting set when a
   conversion operation was started with coding system `undecided' and the
   correct coding system was then detected.)  #### This suggests implementing
   compound text extended segments by saving the state of the ctext stream,
   and installing an appropriate for the duration of the segment.

   IMPORTANT NOTE: It's important, when writing a coding system type, to
   keep straight which type of data goes where.  In particular, `struct
   foo_coding_system' is attached to the coding system object itself.  This
   is a permanent object and there's only one per coding system.  It's
   created once, usually at init time, and never destroyed.  So, `struct
   foo_coding_system' should in general not contain dynamic data! (Just
   data describing the properties of the coding system.) In particular,
   *NO* data about any conversion in progress.  There may be many
   conversions going on simultaneously using a particular coding system,
   and by storing conversion data in the coding system, these conversions
   will overwrite each other's data.

   Instead, use the lstream object, whose purpose is to encapsulate a
   particular conversion and all associated data.  From the lstream object,
   you can get the struct coding_stream using something like

   struct coding_stream *str = LSTREAM_TYPE_DATA (lstr, coding);

   But usually this structure is already passed to you as one of the
   parameters of the method being invoked.

   From the struct coding_stream, you can retrieve the
   coding-system-type-specific data using something like

   struct foo_coding_stream *data = CODING_STREAM_TYPE_DATA (str, foo);

   Then, use this structure to hold all data relevant to the particular
   conversion being done.

   Initialize this structure whenever init_coding_stream_method is called
   (this may happen more than once), and finalize it (free resources, etc.)
   when finalize_coding_stream_method is called.

   Here is a diagram of the structures:

   +------------------------------------------------------------------+
   |                                                                  |
   |                                    +---------------------------+ |
   | struct Lisp_Coding_System          |                           | |
   |                                    | struct foo_coding_system  | |
   |                                    |                           | |
   |                                    +---------------------------+ |
   +------------------------------------------------------------------+



   +------------------------------------------------------------------+
   |                                                                  |
   |                            +-----------------------------------+ |
   |                            |                            +-----+| |
   |    struct lstream          |    struct coding_stream    | PTR || |
   |                            |                            +-----+| |
   |                            +-------------------------------|---+ |
   |                                                            |     |
   +------------------------------------------------------------|-----+
                                                                |
                                                              \ | /
                                                               \|/
                                                                v
                                        +---------------------------+
                                        |                           |
                                        | struct foo_coding_stream  |
                                        |                           |
                                        +---------------------------+


   +-------------------------------------------------------------------------+
   |                   struct detection_state                                |
   |                                                                         |
   | data_offset[detector_foo]:   data_offset[detector_bar]:                 |
   | +-----------------------+    +------------------------+                 |
   | |                       |    |                        |                 |
   | |  struct foo_detector  |    |  struct bar_detector   |     etc.        |
   | |                       |    |                        |                 |
   | +-----------------------+    +------------------------+                 |
   |                                                                         |
   +-------------------------------------------------------------------------+

  --ben
*/

struct coding_stream;
struct detection_state;

extern const struct sized_memory_description coding_system_methods_description;

struct coding_system_methods;

enum source_sink_type
{
  DECODES_CHARACTER_TO_BYTE,
  DECODES_BYTE_TO_BYTE,
  DECODES_BYTE_TO_CHARACTER,
  DECODES_CHARACTER_TO_CHARACTER
};

enum eol_type
{
  EOL_LF,
  EOL_CRLF,
  EOL_CR,
  EOL_AUTODETECT
};

enum handle_coding_error
{
  CODING_RETURN_IGNORE_INVALID_SEQUENCE,
  CODING_RETURN,
  CODING_CONTINUE,
};

struct Lisp_Coding_System
{
  NORMAL_LISP_OBJECT_HEADER header;
  struct coding_system_methods *methods;

#define CODING_SYSTEM_SLOT_DECLARATION
#define MARKED_SLOT(x) Lisp_Object x;
#include "coding-system-slots.h"

  /* Eol type requested by user.  See comment about EOL junk in
     coding-system-slots.h. */
  enum eol_type eol_type;

  /* If true, this is an internal coding system, which will not show up in
     coding-system-list unless a special parameter is given to it. */
  int internal_p;

  /* type-specific extra data attached to a coding_system */
  char data[1];
};
typedef struct Lisp_Coding_System Lisp_Coding_System;

DECLARE_LISP_OBJECT (coding_system, Lisp_Coding_System);
#define XCODING_SYSTEM(x) XRECORD (x, coding_system, Lisp_Coding_System)
#define wrap_coding_system(p) wrap_record (p, coding_system)
#define CODING_SYSTEMP(x) RECORDP (x, coding_system)
#define CHECK_CODING_SYSTEM(x) CHECK_RECORD (x, coding_system)
#define CONCHECK_CODING_SYSTEM(x) CONCHECK_RECORD (x, coding_system)

enum coding_system_variant
{
  no_conversion_coding_system,
  convert_eol_coding_system,
  undecided_coding_system,
  chain_coding_system,
  text_file_wrapper_coding_system,
  internal_coding_system,
  gzip_coding_system,
  mswindows_multibyte_to_unicode_coding_system,
  mswindows_multibyte_coding_system,
  iso2022_coding_system,
  ccl_coding_system,
  shift_jis_coding_system,
  big5_coding_system,
  unicode_coding_system,
  multibyte_coding_system,
};

struct coding_system_methods
{
  Lisp_Object type;
  Lisp_Object predicate_symbol;

  /* Type expressed as an enum, needed for KKCC marking of the
     type-specific lstream data; copied into the struct coding_stream. */

  enum coding_system_variant enumtype;

  /* Implementation specific methods: */

  /* Init method: Initialize coding-system data.  Optional. */
  void (*init_method) (Lisp_Object coding_system);

  /* Print method: Print the type-specific properties of this coding
     system, as part of `print'-ing the object.  If this method is defined
     and prints anything, it should print a space as the first thing it
     does.  Optional. */
  void (*print_method) (Lisp_Object cs, Lisp_Object printcharfun,
			int escapeflag);

  /* Canonicalize method: Convert this coding system to another one; called
     once, at creation time, after all properties have been parsed.  The
     returned value should be a coding system created with
     make_internal_coding_system() (passing the existing coding system as the
     first argument), and will become the coding system returned by
     `make-coding-system'. Optional.

     NOTE: There are *three* different uses of "canonical" or "canonicalize"
     w.r.t. coding systems, and it's important to keep them straight.

     1. The canonicalize method.  Used to specify a different coding
        system, used when doing conversions, in place of the actual coding
        system itself.  Stored in the CANONICAL field of a coding system.

     2. The canonicalize-after-coding method.  Used to return the encoding
        that was "actually" used to decode some text, such that this
	particular encoding can be used to encode the text again with the
	expectation that the result will be the same as the original encoding.
	Particularly important with auto-detecting coding systems.

     3. From the perspective of aliases, a "canonical" coding system is one
        that's not an alias to some other coding system, and "canonicalization"
	is the process of traversing the alias pointers to find the canonical
	coding system that's equivalent to the alias.
     */
  Lisp_Object (*canonicalize_method) (Lisp_Object coding_system);

  /* Canonicalize after coding method: Convert this coding system to
     another one, after coding (usually decoding) has finished.  This is
     meant to be used by auto-detecting coding systems, which should return
     the actually detected coding system.  Optional. */
  Lisp_Object (*canonicalize_after_coding_method)
    (struct coding_stream *str);

  /* Convert method: Decode or encode the data in SRC of size N, writing
     the results into the Dynarr DST.  If the conversion_end_type method
     indicates that the source is characters (as opposed to bytes), you are
     guaranteed to get only whole characters in the data in SRC/N.  STR, a
     struct coding_stream, stores all necessary state and other info about
     the conversion.  Coding-specific state (struct TYPE_coding_stream) can
     be retrieved from STR using CODING_STREAM_TYPE_DATA().  Return value
     indicates the number of bytes of the *INPUT* that were converted (not
     the number of bytes written to the Dynarr!).  This can be less than
     the total amount of input passed in; if so, the remainder is
     considered "rejected" and will appear again at the beginning of the
     data passed in the next time the convert method is called.  When EOF
     is returned on the other end and there's no more data, the convert
     method will be called one last time, STR->eof set and the passed-in
     data will consist only of any rejected data from the previous
     call. (At this point, file handles and similar resources can be
     closed, but do NOT arbitrarily free data structures in the
     type-specific data, because there are operations that can be done on
     closed streams to query the results of the processing -- specifically,
     for coding streams, there's the canonicalize_after_coding() method.)
     Required. */
  Bytecount (*convert_method) (struct coding_stream *str,
			       const unsigned char *src, Bytecount n,
			       unsigned_char_dynarr *dst);

  /* Init coding stream method: Initialize the type-specific data attached
     to the coding stream (i.e. in struct TYPE_coding_stream), when the
     coding stream is opened.  This may be called multiple times -- in
     particular, upon rewinding a coding stream, if no
     `rewind_coding_stream' method is called, `init_coding_stream' will be
     called.  The first time this is called, the type-specific data will be
     zeroed out, but not when called as a result of a rewind().  Optional. */
  void (*init_coding_stream_method) (struct coding_stream *str);

  /* Rewind coding stream method: Reset any necessary type-specific data as
     a result of the stream being rewound.  Optional.  If not specified,
     the init_coding_stream method will be called upon rewind. */
  void (*rewind_coding_stream_method) (struct coding_stream *str);

  /* Return the number of characters *decoded*. Optional. */
  Charcount (*character_tell_method) (struct coding_stream *str);

  /* Copy coding stream method: Copy any indirect pointers from SRC to DST.
     SRC and DST are pointers to the type-specific coding-stream data (e.g.
     `struct iso2022_coding_stream').  At the time this function is called,
     a destination structure has already been allocated and a bitwise copy
     performed from the source structure to the destination structure.
     The purpose of the method is to deal with indirect structures that need
     to be allocated to avoid being shared between the two structures.  This
     method is used when doing the equivalent of tell(): To remember how to
     seek back to a previous position, we have to remember the coding state
     at that position. */
  void (*copy_coding_stream_method) (void *dst, void *src);

  /* Finalize coding stream method: Clean up the type-specific data
     attached to the coding stream (i.e. in struct TYPE_coding_stream).
     Happens when the Lstream is deleted using Lstream_delete() or is
     garbage-collected.  Most streams are deleted after they've been used,
     so it's less likely (but still possible) that allocated data will
     stick around until GC time. (File handles can also be closed when EOF
     is signalled; but some data must stick around after this point, for
     the benefit of canonicalize_after_coding.  See the convert method.)
     Called only once.  Optional. */
  void (*finalize_coding_stream_method) (struct coding_stream *str);

  /* Finalize method: Clean up type-specific data (e.g. free allocated
     data) attached to the coding system (i.e. in struct
     TYPE_coding_system), when the coding system is about to be garbage
     collected. (Currently not called.) Called only once.  Optional. */
  void (*finalize_method) (Lisp_Object codesys);

  /* Conversion end type method: Does this coding system encode bytes ->
     characters, characters -> characters, bytes -> bytes, or
     characters -> bytes?.  Default is characters -> bytes.  Optional. */
  enum source_sink_type (*conversion_end_type_method) (Lisp_Object codesys);

  /* Putprop method: Set the value of a type-specific property.  If
     the property name is unrecognized, return 0.  If the value is disallowed
     or erroneous, signal an error.  Currently called only at creation time.
     Optional. */
  int (*putprop_method) (Lisp_Object codesys,
			 Lisp_Object key,
			 Lisp_Object value);

  /* Getprop method: Return the value of a type-specific property.  If
     the property name is unrecognized, return Qunbound.  Optional.
   */
  Lisp_Object (*getprop_method) (Lisp_Object coding_system,
				 Lisp_Object prop);

  /* These next three are set as part of the call to
     INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA. */

  /* Description of the extra data (struct foo_coding_system) attached to a
     coding system, for pdump/KKCC purposes. */
  const struct sized_memory_description *extra_description;
  /* size of struct foo_coding_system -- extra data associated with
     the coding system */
  Bytecount extra_data_size;
  /* Description of the extra data (struct foo_coding_stream) attached to an
     lstream, for KKCC purposes. */
  const struct sized_memory_description *stream_description;
  /* size of struct foo_coding_stream -- extra data associated with the
     struct coding_stream, needed for each active coding process
     using this coding system.  note that we can have more than one
     process active at once (simply by creating more than one coding
     lstream using this coding system), so we can't store this data in
     the coding system object. */
  Bytecount stream_data_size;
};

enum coding_error
  {
    CODING_SUCCEEDED,
    CODING_UNENCODABLE,
    CODING_INVALID_SEQUENCE,
  };

extern Lisp_Object Qquery_coding_warning_face;

/***** Calling a coding-system method *****/

#define RAW_CODESYSMETH(cs, m) ((cs)->methods->m##_method)
#define HAS_CODESYSMETH_P(cs, m) (!!RAW_CODESYSMETH (cs, m))
#define CODESYSMETH(cs, m, args) (((cs)->methods->m##_method) args)

/* Call a void-returning coding-system method, if it exists.  */
#define MAYBE_CODESYSMETH(cs, m, args) do {	\
  Lisp_Coding_System *maybe_codesysmeth_cs = (cs);	\
  if (HAS_CODESYSMETH_P (maybe_codesysmeth_cs, m))	\
    CODESYSMETH (maybe_codesysmeth_cs, m, args);	\
} while (0)

/* Call a coding-system method, if it exists, or return GIVEN.
   NOTE: Multiply-evaluates CS. */
#define CODESYSMETH_OR_GIVEN(cs, m, args, given)	\
  (HAS_CODESYSMETH_P (cs, m) ?				\
   CODESYSMETH (cs, m, args) : (given))

#define HAS_XCODESYSMETH_P(cs, m) HAS_CODESYSMETH_P (XCODING_SYSTEM (cs), m)
#define XCODESYSMETH(cs, m, args) \
  CODESYSMETH (XCODING_SYSTEM (cs), m, args)
#define MAYBE_XCODESYSMETH(cs, m, args) \
  MAYBE_CODESYSMETH (XCODING_SYSTEM (cs), m, args)
#define XCODESYSMETH_OR_GIVEN(cs, m, args, given) \
  CODESYSMETH_OR_GIVEN (XCODING_SYSTEM (cs), m, args, given)

/***** Defining new coding-system types *****/

extern const struct sized_memory_description coding_system_empty_extra_description;

#ifdef ERROR_CHECK_TYPES
#define DECLARE_CODING_SYSTEM_TYPE(type)				\
									\
extern struct coding_system_methods * type##_coding_system_methods;	\
DECLARE_INLINE_HEADER (							\
struct type##_coding_system *						\
error_check_##type##_coding_system_data (Lisp_Coding_System *cs)	\
)									\
{									\
  assert (CODING_SYSTEM_TYPE_P (cs, type));				\
  /* Catch accidental use of INITIALIZE_CODING_SYSTEM_TYPE in place	\
     of INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA. */			\
  assert (cs->methods->extra_data_size > 0);				\
  return (struct type##_coding_system *) cs->data;			\
}									\
									\
DECLARE_INLINE_HEADER (							\
struct type##_coding_stream *						\
error_check_##type##_coding_stream_data (struct coding_stream *s)	\
)									\
{									\
  assert (XCODING_SYSTEM_TYPE_P (s->codesys, type));			\
  return (struct type##_coding_stream *) s->st.data;			\
}									\
									\
DECLARE_INLINE_HEADER (							\
Lisp_Coding_System *							\
error_check_##type##_coding_system_type (Lisp_Object obj)		\
)									\
{									\
  Lisp_Coding_System *cs = XCODING_SYSTEM (obj);			\
  assert (CODING_SYSTEM_TYPE_P (cs, type));				\
  return cs;								\
}									\
									\
DECLARE_NOTHING
#else
#define DECLARE_CODING_SYSTEM_TYPE(type)				\
extern struct coding_system_methods * type##_coding_system_methods
#endif /* ERROR_CHECK_TYPES */

#define DEFINE_CODING_SYSTEM_TYPE(type)					\
struct coding_system_methods * type##_coding_system_methods

#define DEFINE_CODING_SYSTEM_TYPE_WITH_DATA(type)		\
struct coding_system_methods * type##_coding_system_methods;	\
static const struct sized_memory_description			\
  type##_coding_system_description_0 = {			\
  sizeof (struct type##_coding_system),				\
  type##_coding_system_description				\
};								\
static const struct sized_memory_description			\
  type##_coding_stream_description_0 = {			\
  sizeof (struct type##_coding_stream),				\
  type##_coding_stream_description				\
}

#define INITIALIZE_CODING_SYSTEM_TYPE(ty, pred_sym) do {		\
  ty##_coding_system_methods =						\
    xnew_and_zero (struct coding_system_methods);			\
  ty##_coding_system_methods->type = Q##ty;				\
  ty##_coding_system_methods->extra_description =			\
    &coding_system_empty_extra_description;				\
  ty##_coding_system_methods->enumtype = ty##_coding_system;		\
  defsymbol (&ty##_coding_system_methods->predicate_symbol, pred_sym);	\
  add_entry_to_coding_system_type_list (ty##_coding_system_methods);	\
  dump_add_root_block_ptr (&ty##_coding_system_methods,		\
                            &coding_system_methods_description);	\
} while (0)

/* This assumes the existence of two structures:

   struct foo_coding_system (attached to the coding system)
   struct foo_coding_stream (per coding process, attached to the
                             struct coding_stream)
   const struct memory_description foo_coding_system_description[]
   (data description of struct foo_coding_system)

   For an example of how to do the description, see
   chain_coding_system_description.
*/
#define INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA(type, pred_sym)	\
do {								\
  INITIALIZE_CODING_SYSTEM_TYPE (type, pred_sym);		\
  type##_coding_system_methods->extra_description =		\
    &type##_coding_system_description_0;			\
  type##_coding_system_methods->extra_data_size =		\
    sizeof (struct type##_coding_system);			\
  type##_coding_system_methods->stream_description =		\
    &type##_coding_stream_description_0;			\
  type##_coding_system_methods->stream_data_size =		\
    sizeof (struct type##_coding_stream);			\
} while (0)

/* Declare that coding-system-type TYPE has method METH; used in
   initialization routines */
#define CODING_SYSTEM_HAS_METHOD(type, meth) \
  (type##_coding_system_methods->meth##_method = type##_##meth)

/***** Macros for accessing coding-system types *****/

#define CODING_SYSTEM_TYPE_P(cs, type) \
  ((cs)->methods == type##_coding_system_methods)
#define XCODING_SYSTEM_TYPE_P(cs, type) \
  CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (cs), type)

#ifdef ERROR_CHECK_TYPES
# define CODING_SYSTEM_TYPE_DATA(cs, type) \
   error_check_##type##_coding_system_data (cs)
#else
# define CODING_SYSTEM_TYPE_DATA(cs, type)		\
  ((struct type##_coding_system *)			\
    (cs)->data)
#endif

#define XCODING_SYSTEM_TYPE_DATA(cs, type) \
  CODING_SYSTEM_TYPE_DATA (XCODING_SYSTEM_OF_TYPE (cs, type), type)

#ifdef ERROR_CHECK_TYPES
# define XCODING_SYSTEM_OF_TYPE(x, type)	\
   error_check_##type##_coding_system_type (x)
# define XSETCODING_SYSTEM_OF_TYPE(x, p, type)	do		\
{								\
  x = wrap_coding_system (p);					\
  assert (CODING_SYSTEM_TYPEP (XCODING_SYSTEM (x), type));	\
} while (0)
#else
# define XCODING_SYSTEM_OF_TYPE(x, type) XCODING_SYSTEM (x)
# define XSETCODING_SYSTEM_OF_TYPE(x, p, type) do	\
{							\
  x = wrap_coding_system (p);				\
} while (0)
#endif /* ERROR_CHECK_TYPE_CHECK */

#define CODING_SYSTEM_TYPEP(x, type)					  \
  (CODING_SYSTEMP (x) && CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (x), type))
#define CHECK_CODING_SYSTEM_OF_TYPE(x, type) do {			\
  CHECK_CODING_SYSTEM (x);					\
  if (!CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (x), type))		\
    dead_wrong_type_argument					\
      (type##_coding_system_methods->predicate_symbol, x);	\
} while (0)
#define CONCHECK_CODING_SYSTEM_OF_TYPE(x, type) do {		\
  CONCHECK_CODING_SYSTEM (x);					\
  if (!(CODING_SYSTEM_TYPEP (x, type)))				\
    x = wrong_type_argument					\
      (type##_coding_system_methods->predicate_symbol, x);	\
} while (0)

#define CODING_SYSTEM_METHODS(codesys) ((codesys)->methods)
#define CODING_SYSTEM_NAME(codesys) ((codesys)->name)
#define CODING_SYSTEM_DESCRIPTION(codesys) ((codesys)->description)
#define CODING_SYSTEM_TYPE(codesys) ((codesys)->methods->type)
#define CODING_SYSTEM_MNEMONIC(codesys) ((codesys)->mnemonic)
#define CODING_SYSTEM_DOCUMENTATION(codesys) ((codesys)->documentation)
#define CODING_SYSTEM_POST_READ_CONVERSION(codesys) \
  ((codesys)->post_read_conversion)
#define CODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
  ((codesys)->pre_write_conversion)
#define CODING_SYSTEM_EOL_TYPE(codesys) ((codesys)->eol_type)
#define CODING_SYSTEM_EOL_LF(codesys)   ((codesys)->eol[EOL_LF])
#define CODING_SYSTEM_EOL_CRLF(codesys) ((codesys)->eol[EOL_CRLF])
#define CODING_SYSTEM_EOL_CR(codesys)   ((codesys)->eol[EOL_CR])
#define CODING_SYSTEM_TEXT_FILE_WRAPPER(codesys) ((codesys)->text_file_wrapper)
#define CODING_SYSTEM_AUTO_EOL_WRAPPER(codesys) ((codesys)->auto_eol_wrapper)
#define CODING_SYSTEM_SUBSIDIARY_PARENT(codesys) ((codesys)->subsidiary_parent)
#define CODING_SYSTEM_CANONICAL(codesys) ((codesys)->canonical)
#define CODING_SYSTEM_SAFE_CHARSETS(codesys) ((codesys)->safe_charsets)
#define CODING_SYSTEM_SAFE_CHARS(codesys) ((codesys)->safe_chars)

#define CODING_SYSTEM_CHAIN_CHAIN(codesys) \
  (CODING_SYSTEM_TYPE_DATA (codesys, chain)->chain)
#define CODING_SYSTEM_CHAIN_COUNT(codesys) \
  (CODING_SYSTEM_TYPE_DATA (codesys, chain)->count)
#define CODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING(codesys) \
  (CODING_SYSTEM_TYPE_DATA (codesys, chain)->canonicalize_after_coding)

#define XCODING_SYSTEM_METHODS(codesys) \
  CODING_SYSTEM_METHODS (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_NAME(codesys) \
  CODING_SYSTEM_NAME (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_DESCRIPTION(codesys) \
  CODING_SYSTEM_DESCRIPTION (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_TYPE(codesys) \
  CODING_SYSTEM_TYPE (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_MNEMONIC(codesys) \
  CODING_SYSTEM_MNEMONIC (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_DOCUMENTATION(codesys) \
  CODING_SYSTEM_DOCUMENTATION (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_POST_READ_CONVERSION(codesys) \
  CODING_SYSTEM_POST_READ_CONVERSION (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
  CODING_SYSTEM_PRE_WRITE_CONVERSION (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_EOL_TYPE(codesys) \
  CODING_SYSTEM_EOL_TYPE (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_EOL_LF(codesys) \
  CODING_SYSTEM_EOL_LF (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_EOL_CRLF(codesys) \
  CODING_SYSTEM_EOL_CRLF (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_EOL_CR(codesys) \
  CODING_SYSTEM_EOL_CR (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_TEXT_FILE_WRAPPER(codesys) \
  CODING_SYSTEM_TEXT_FILE_WRAPPER (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_AUTO_EOL_WRAPPER(codesys) \
  CODING_SYSTEM_AUTO_EOL_WRAPPER (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_SUBSIDIARY_PARENT(codesys) \
  CODING_SYSTEM_SUBSIDIARY_PARENT (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_CANONICAL(codesys) \
  CODING_SYSTEM_CANONICAL (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_SAFE_CHARSETS(codesys) \
  CODING_SYSTEM_SAFE_CHARSETS (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_SAFE_CHARS(codesys) \
  CODING_SYSTEM_SAFE_CHARS (XCODING_SYSTEM (codesys))

#define XCODING_SYSTEM_CHAIN_CHAIN(codesys) \
  CODING_SYSTEM_CHAIN_CHAIN (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_CHAIN_COUNT(codesys) \
  CODING_SYSTEM_CHAIN_COUNT (XCODING_SYSTEM (codesys))
#define XCODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING(codesys) \
  CODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING (XCODING_SYSTEM (codesys))

/**************************************************/
/*                   Detection                    */
/**************************************************/

#define MAX_DETECTOR_CATEGORIES 256
#define MAX_DETECTORS 64

#define MAX_BYTES_PROCESSED_FOR_DETECTION 65536

struct detection_state
{
  NORMAL_LISP_OBJECT_HEADER header;

  int seen_non_ascii;
  Bytecount bytes_seen;

  char categories[MAX_DETECTOR_CATEGORIES];
  Bytecount data_offset[MAX_DETECTORS];
  /* ... more data follows; data_offset[detector_##TYPE] points to
     the data for that type */
};

DECLARE_LISP_OBJECT (detection_state, struct detection_state);
#define XDETECTION_STATE(x) XRECORD (x, detection_state, struct detection_state)
#define wrap_detection_state(p) wrap_record (p, detection_state)
#define DETECTION_STATEP(x) RECORDP (x, detection_state)
#define CHECK_DETECTION_STATE(x) CHECK_RECORD (x, detection_state)
#define CONCHECK_DETECTION_STATE(x) CONCHECK_RECORD (x, detection_state)

#define DETECTION_STATE_DATA(st, type)				\
  ((struct type##_detector *)					\
   ((char *) (st) + (st)->data_offset[detector_##type]))

enum detection_result
 {
  /* Basically means a magic cookie was seen indicating this type, or
     something similar. */
  DET_NEAR_CERTAINTY = 4,
  DET_HIGHEST = 4,
  /* Characteristics seen that are unlikely to be other coding system types
     -- e.g. ISO-2022 escape sequences, or perhaps a consistent pattern of
     alternating zero bytes in UTF-16, along with Unicode LF or CRLF
     sequences at regular intervals. (Zero bytes are unlikely or impossible
     in most text encodings.) */
  DET_QUITE_PROBABLE = 3,
  /* Strong or medium statistical likelihood.  At least some
     characteristics seen that match what's normally found in this encoding
     -- e.g. in Shift-JIS, a number of two-byte Japanese character
     sequences in the right range, and nothing out of range; or in Unicode,
     much higher statistical variance in the odd bytes than in the even
     bytes, or vice-versa (perhaps the presence of regular EOL sequences
     would bump this too to DET_QUITE_PROBABLE).  This is quite often a
     statistical test. */
  DET_SOMEWHAT_LIKELY = 2,
  /* Weak statistical likelihood.  Pretty much any features at all that
     characterize this encoding, and nothing that rules against it. */
  DET_SLIGHTLY_LIKELY = 1,
  /* Default state.  Perhaps it indicates pure ASCII or something similarly
     vague seen in Shift-JIS, or, exactly as the level says, it might mean
     in a statistical-based detector that the pros and cons are balanced
     out.  This is also the lowest level that will be accepted by the
     auto-detector without asking the user: If all available detectors
     report lower levels for all categories with attached coding systems,
     the user will be shown the results and explicitly prompted for action.
     The user will also be prompted if this is the highest available level
     and more than one detector reports the level. (See below about the
     consequent necessity of an "ASCII" detector, which will return level 1
     or higher for most plain text files.) */
  DET_AS_LIKELY_AS_UNLIKELY = 0,
  /* Some characteristics seen that are unusual for this encoding --
     e.g. unusual control characters in a plain-text encoding, lots of
     8-bit characters, or little statistical variance in the odd and even
     bytes in UTF-16. */
  DET_SOMEWHAT_UNLIKELY = -1,
  /* This indicates that there is very little chance the data is in the
     right format; this is probably the lowest level you can get when
     presenting random binary data to a text file, because there are no
     "specific sequences" you can see that would totally rule out
     recognition. */
  DET_QUITE_IMPROBABLE = -2,
  /* An erroneous sequence was seen. */
  DET_NEARLY_IMPOSSIBLE = -3,
  DET_LOWEST = -3
 };

struct detector_category
{
  int id;
  Lisp_Object sym;
};

typedef struct
{
  Dynarr_declare (struct detector_category);
} detector_category_dynarr;

struct detector
{
  int id;
  detector_category_dynarr *cats;

  /* Detect method: Required. */
  void (*detect_method) (struct detection_state *st,
			 const unsigned char *src, Bytecount n);

  /* Finalize detection state method: Clean up any allocated data in the
     detection state.  Called only once. Optional. */
  void (*finalize_detection_state_method) (struct detection_state *st);

  /* Size of struct foo_detection_state -- data associated with the
     detector, during detection.  Note that `struct foo_detection_state'
     forms part of `struct detection_state' -- for a particular
     struct detection_state `st', it's located at the offset
     st->data_offset[detector_foo].
     */
  Bytecount data_size;
};

/* Lvalue for a particular detection result -- detection state ST,
   category CAT */
#define DET_RESULT(st, cat) ((st)->categories[detector_category_##cat])
/* In state ST, set all detection results associated with detector DET to
   RESULT. */
#define SET_DET_RESULTS(st, det, result) \
  set_detection_results (st, detector_##det, result)

typedef struct
{
  Dynarr_declare (struct detector);
} detector_dynarr;

extern detector_dynarr *all_coding_detectors;

extern int coding_detector_count;
extern int coding_detector_category_count;

#define DEFINE_DETECTOR_CATEGORY(detector, cat) \
int detector_category_##cat

#define DECLARE_DETECTOR_CATEGORY(detector, cat) \
extern int detector_category_##cat

#define INITIALIZE_DETECTOR_CATEGORY(detector, cat)			  \
do {									  \
  struct detector_category dog;						  \
  xzero (dog);								  \
  detector_category_##cat = coding_detector_category_count++;		  \
  dump_add_opaque_int (&detector_category_##cat);			  \
  dog.id = detector_category_##cat;					  \
  dog.sym = Q##cat;							  \
  Dynarr_add (Dynarr_at (all_coding_detectors, detector_##detector).cats, \
	      dog);							  \
} while (0)

#define DEFINE_DETECTOR(Detector) \
int detector_##Detector

/* Here we define a detector that has Lisp objects in it.  We need to
   create a sized_memory_description that points to the memory description
   for the detector-specific detection state (which is the responsibility
   of the creator of the detector). */

#define DEFINE_DETECTOR_WITH_DESCRIPTION(Detector)	\
DEFINE_DETECTOR (Detector);				\
static const struct sized_memory_description		\
   Detector##_detector_description_0 =			\
  { sizeof (struct Detector##_detector),		\
    Detector##_detector_description }

#define DECLARE_DETECTOR(Detector) \
extern int detector_##Detector

/* Initialize a detector without Lisp objects in it.  This creates a blank
   `struct detector' and adds it to the dynarr that tracks all detectors
   so far known about.  We also need to initialize the variable
   `detector_foo' with the index of the detector. */

#define INITIALIZE_DETECTOR(Detector)			\
do {							\
  struct detector det;					\
  xzero (det);						\
  detector_##Detector = coding_detector_count++;	\
  dump_add_opaque_int (&detector_##Detector);		\
  det.id = detector_##Detector;				\
  det.cats = Dynarr_new2 (detector_category_dynarr,	\
			  struct detector_category);	\
  det.data_size = sizeof (struct Detector##_detector);	\
  Dynarr_add (all_coding_detectors, det);		\
} while (0)

/* Here we initialize a detector that has Lisp objects in it that need to
   be noted for KKCC.  Because `struct detection_state' has a strange
   format -- it has the whole set of detector-specific detection states
   included within it, with an array indicating the offset of each
   detector-specific structure within the whole structure -- we can't simply
   declare a static object description, since there isn't any means of
   handling this weird type of object.  Instead, we must build up the
   description bit-by-bit based at loadup time on individual descriptions of
   the detector-specific structures. */
extern void
initialize_detector_description (int detector_id,
                                 const struct sized_memory_description *);

#define INITIALIZE_DETECTOR_WITH_DESCRIPTION(Detector)			\
do {									\
  INITIALIZE_DETECTOR (Detector);					\
  initialize_detector_description (detector_##Detector,                 \
                                   &Detector##_detector_description_0); \
} while (0)

#define DETECTOR_HAS_METHOD(Detector, Meth)				\
  Dynarr_at (all_coding_detectors, detector_##Detector).Meth##_method =	\
    Detector##_##Meth


/**************************************************/
/*               Decoding/Encoding                */
/**************************************************/

/* Is the source (SOURCEP == 1) or sink (SOURCEP == 0) when encoding specified
   in characters? */

enum source_or_sink
{
  CODING_SOURCE,
  CODING_SINK
};

enum encode_decode
{
  CODING_ENCODE,
  CODING_DECODE
};

struct coding_stream_state
{
  /********** State of the conversion currently taking place ************/

  /* Coding-system-specific data holding extra state about the
     conversion.  Logically a struct TYPE_coding_stream; a pointer
     to such a struct, with (when ERROR_CHECK_TYPES is defined)
     error-checking that this is really a structure of that type
     (checking the corresponding coding system type) can be retrieved using
     CODING_STREAM_TYPE_DATA().  Allocated at the same time that
     CODESYS is set (which may occur at any time, even multiple times,
     during the lifetime of the stream).  The size comes from
     methods->stream_data_size.  */
  void *data;

  /********* Fields used to implement error handling **********/

  /* (1) Fields automatically set during the conversion process */
  /* What to do when an error is encountered: Currently only using during
     encoding (during decoding, just write an error-octet into the Ichar
     stream). */
  enum handle_coding_error handle;
  /* Length of destination dynarr prior to conversion. */
  Bytecount written_before_convert;
  /* Total amount written by converter to output dynarr (after both
     non-erroneous and erroneous output written); this is computed by
     subtracting the value of `written_before_convert' from the length of
     the output dynarr. */
  Bytecount total_written;
  /* Total amount of input processed by converter, including erroneous
     input. */
  Bytecount total_read;

  /* (2) Fields set by the convert method when an error occurs. */
  /* Total amount of non-erroneous input processed by converter */
  Bytecount good_read;
  /* Amount of non-erroneous data written by converter to output dynarr;
     this is computed by subtracting the value of
     `written_before_convert' from the length of the output dynarr after
     the non-erroneous data is written but before the erroneous data is
     written. */
  Bytecount good_written;
  /* Type of error that occurred.  Note that this is automatically set to
     CODING_SUCCEEDED before the convert method is called, so it only needs
     to be set when an error occurs. */
  enum coding_error error_occurred;

  /* (3) Fields used in converting the above values into cumulative values
         (offsets from the beginning of of the stream, rather than the
         beginning of this chunk of data) */

  /* Cumulative including most recent call to conversion method (i.e.
     including the data up through TOTAL_WRITTEN and TOTAL_READ). */
  Bytecount new_cumul_read, new_cumul_written;
  /* Previous values of same -- not including most recent call to conversion
     method. */
  Bytecount cumul_read, cumul_written;


  /********* Miscellaneous flags **********/

  /* If set, this is the last chunk of data being processed.  When this is
     finished, output any necessary terminating control characters, escape
     sequences, etc. */
  unsigned int eof :1;

  /* #### Temporary test to verify that finalizers don't ever get called
     more than once. */
  unsigned int finalized :1;
};

/* Data structure attached to an lstream of type `coding',
   containing values specific to the coding process.  Additional
   data is stored in the DATA field below; the exact form of that data
   is controlled by the type of the coding system that governs the
   conversion (field CODESYS).  CODESYS may be set at any time
   throughout the lifetime of the lstream and possibly more than once.
   See long comment above for more info. */

struct coding_stream
{
  /************ Back pointers to the coding system ************/

  /* Enumerated constant listing which type of coding system this is.  This
     duplicates the method structure in XCODING_SYSTEM
     (str->codesys)->methods->type, which formerly was the only way to
     determine the coding system type.  We need this constant now for KKCC,
     so that it can be used in an XD_UNION clause to determine the Lisp
     objects in the type-specific data. */
  enum coding_system_variant type;

  /* Coding system that governs the conversion. */
  Lisp_Object codesys;
  /* Original coding system, pre-canonicalization. */
  Lisp_Object orig_codesys;

  /************ Pointers to the associated stream(s) ************/

  /* Back pointer to current stream. */
  Lstream *us;

  /* Stream that we read the unprocessed data from or write the processed
     data to. */
  Lstream *other_end;

  /********* Internal dynarrs used to handle buffering/slippage **********/

  /* In order to handle both reading to and writing from a coding stream,
     we phrase the conversion methods like write methods -- we can
     implement reading in terms of a write method but not vice-versa,
     because the write method is forced to take only what it's given but
     the read method can read more data from the other end if necessary.
     On the other hand, the write method is free to generate all the data
     it wants (and just write it to the other end), but the read method
     can return only as much as was asked for, so we need to implement our
     own buffering. */

  /* If we are reading, then we can return only a fixed amount of data, but
     the converter is free to return as much as it wants, so we direct it
     to store the data here and lop off chunks as we need them.  If we are
     writing, we use this because the converter takes a Dynarr but we are
     supposed to write into a fixed buffer. (NOTE: This introduces an extra
     memory copy.) */
  unsigned_char_dynarr *convert_to;

  /* The conversion method might reject some of the data -- this typically
     includes partial characters, partial escape sequences, etc.  When
     writing, we just pass the rejection up to the Lstream module, and it
     will buffer the data.  When reading, however, we need to do the
     buffering ourselves, and we put it here, combined with newly read
     data. */
  unsigned_char_dynarr *convert_from;

  /********* Information about the encoding taking place **********/

  /* Direction of encoding */
  enum encode_decode direction;

  /* Source and length of data to convert.  This duplicates information
     passed in to the conversion method as parameters, and is placed here
     so that the conversion methods don't have to remember the original
     values of these parameters. */
  const unsigned char *src;
  Bytecount srclen;

  /* Current state of the conversion process.  This should contain
     everything state-related, so that copying the contents of this
     structure (and any indirect pointers) at a particular point in the
     stream and later on copying them back should allow seeking back to
     that location. */
  struct coding_stream_state st;
};

#define CODING_STREAM_DATA(stream) LSTREAM_TYPE_DATA (stream, coding)

#ifdef ERROR_CHECK_TYPES
# define CODING_STREAM_TYPE_DATA(s, type) \
   error_check_##type##_coding_stream_data (s)
#else
# define CODING_STREAM_TYPE_DATA(s, type) \
  ((struct type##_coding_stream *) (s)->st.data)
#endif

/* Add OCTET as an undecodable literal "error" octet character, stored
   using a 256-byte range in private Unicode space */
#define DECODE_ERROR_OCTET(octet, dst) \
  decode_unicode_to_dynarr ((octet) + UNICODE_ERROR_OCTET_RANGE_START, dst)

#define DECODE_OUTPUT_PARTIAL_CHAR(str, data, dst)	\
do {							\
  if (str->st.eof && data->ch >= 0)			\
    {							\
      DECODE_ERROR_OCTET (data->ch, dst);		\
      data->ch = -1;					\
    }							\
} while (0)

/* We can't use do ... while (0) here, because we need to execute `continue',
   so we can't have a secondary loop surrounding the code, but this if/else
   statement should be safe even if placed in a context like this:

   if (foo)
     ENCODING_ERROR_RETURN_OR_CONTINUE(...);
   else
     do something else;

   because it will expand to

   if (foo)
     if (condition)
       {
         ...
         continue;
       }
     else return (...);
   else
     do something else;
*/
#define ENCODING_ERROR_RETURN_OR_CONTINUE(str, src)			\
if ((str)->st.handle == CODING_CONTINUE ||				\
    ((str)->st.handle == CODING_RETURN_IGNORE_INVALID_SEQUENCE &&	\
     (str)->st.error_occurred == CODING_INVALID_SEQUENCE))		\
  {									\
    (str)->st.error_occurred = CODING_SUCCEEDED;			\
    continue;								\
  }									\
else return ((src) - (str->src))

/* Convert shift-JIS code (sj1, sj2) into JISX0208 position codes (c1, c2). */
#define DECODE_SHIFT_JIS(sj1, sj2, c1, c2)		\
do {							\
  int I1 = sj1, I2 = sj2;				\
  if (I2 >= 0x9f)					\
    c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe0 : 0x60),	\
    c2 = I2 + 2;					\
  else							\
    c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe1 : 0x61),	\
    c2 = I2 + ((I2 >= 0x7f) ? 0x60 : 0x61);		\
  c1 -= 0x80; c2 -= 0x80;				\
} while (0)

/* Convert the JISX0208 position codes (c1, c2) into Shift-JIS code
   (sj1, sj2). */

#define ENCODE_SHIFT_JIS(c1, c2, sj1, sj2)		\
do {							\
  int I1 = c1 + 0x80, I2 = c2 + 0x80;			\
  if (I1 & 1)						\
    sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x31 : 0x71),	\
    sj2 = I2 - ((I2 >= 0xe0) ? 0x60 : 0x61);		\
  else							\
    sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x30 : 0x70),	\
    sj2 = I2 - 2;					\
} while (0)

/* Macros to define code of control characters for ISO2022's functions.  */
/* Used by the detection routines of other coding system types as well. */
			/* code */	/* function */
#define ISO_CODE_LF	0x0A		/* line-feed */
#define ISO_CODE_CR	0x0D		/* carriage-return */
#define ISO_CODE_SO	0x0E		/* shift-out */
#define ISO_CODE_SI	0x0F		/* shift-in */
#define ISO_CODE_ESC	0x1B		/* escape */
#define ISO_CODE_DEL	0x7F		/* delete */
#define ISO_CODE_SS2	0x8E		/* single-shift-2 */
#define ISO_CODE_SS3	0x8F		/* single-shift-3 */
#define ISO_CODE_CSI	0x9B		/* control-sequence-introduce */

DECLARE_CODING_SYSTEM_TYPE (no_conversion);
DECLARE_CODING_SYSTEM_TYPE (convert_eol);
#if 0
DECLARE_CODING_SYSTEM_TYPE (text_file_wrapper);
#endif /* 0 */
DECLARE_CODING_SYSTEM_TYPE (undecided);
DECLARE_CODING_SYSTEM_TYPE (chain);

#ifdef DEBUG_XEMACS
DECLARE_CODING_SYSTEM_TYPE (internal);
#endif

DECLARE_CODING_SYSTEM_TYPE (multibyte);
DECLARE_CODING_SYSTEM_TYPE (iso2022);
DECLARE_CODING_SYSTEM_TYPE (ccl);
DECLARE_CODING_SYSTEM_TYPE (shift_jis);
DECLARE_CODING_SYSTEM_TYPE (big5);

#ifdef HAVE_ZLIB
DECLARE_CODING_SYSTEM_TYPE (gzip);
#endif

DECLARE_CODING_SYSTEM_TYPE (unicode);

#ifdef WIN32_ANY
DECLARE_CODING_SYSTEM_TYPE (mswindows_multibyte_to_unicode);
DECLARE_CODING_SYSTEM_TYPE (mswindows_multibyte);
#endif

Lisp_Object coding_stream_detected_coding_system (Lstream *stream);
Lisp_Object coding_stream_coding_system (Lstream *stream);
void set_coding_stream_coding_system (Lstream *stream,
				      Lisp_Object codesys);
Lisp_Object detect_coding_stream (Lisp_Object stream);
void big5_char_to_fake_codepoint (int b1, int b2, Lisp_Object *charset,
				  int *c1, int *c2);
void add_entry_to_coding_system_type_list (struct coding_system_methods *m);
Lisp_Object make_internal_coding_system (Lisp_Object existing,
					 const Ascbyte *prefix,
					 Lisp_Object type,
					 Lisp_Object description,
					 Lisp_Object props);
#ifdef DEBUG_XEMACS
extern Boolint debug_coding_detection;
#endif /* DEBUG_XEMACS */

/* Additional flags to make_coding_input_stream() and
   make_coding_output_stream() that can be passed at creation time. */
#define LSTR_NO_CLOSE_OTHER				(1 << 16)
#define LSTR_READ_ONE_BYTE_AT_A_TIME			(1 << 17)
#define LSTR_NO_INIT_CHAR_MODE_WHEN_READING		(1 << 18)
#define LSTR_STOP_ON_ERROR				(1 << 19)
#define LSTR_STOP_ON_ERROR_IGNORE_INVALID_SEQUENCE	(1 << 20)

/* Used internally */
#define LSTR_STOP_ON_SOME_ERROR \
  (LSTR_STOP_ON_ERROR | LSTR_STOP_ON_ERROR_IGNORE_INVALID_SEQUENCE)

Lisp_Object make_coding_input_stream (Lstream *stream, Lisp_Object codesys,
				      enum encode_decode direction,
				      int flags);
Lisp_Object make_coding_output_stream (Lstream *stream, Lisp_Object codesys,
				       enum encode_decode direction,
				       int flags);
void coding_stream_get_error_info (Lstream *stream, enum coding_error *errtype,
				   Bytecount *good_source,
				   Bytecount *total_source,
				   Bytecount *good_sink,
				   Bytecount *total_sink);
void set_detection_results (struct detection_state *st, int detector,
			    int given);


void handle_encoding_error_after_output (struct coding_stream *str,
					 const UExtbyte *src,
					 unsigned_char_dynarr *dst,
					 Ichar num_input_err_chars,
					 Bytecount num_output_err_bytes,
					 enum coding_error errtype);
void handle_encoding_error_before_output (struct coding_stream *str,
					  const UExtbyte *src,
					  unsigned_char_dynarr *dst,
					  Ichar num_input_err_chars,
					  enum coding_error errtype);
void handle_standard_encoding_error (struct coding_stream *str,
				     const UExtbyte *src,
				     unsigned_char_dynarr *dst);
int handle_possible_error_octet (Ichar ich,
				 struct coding_stream *str,
				 const UExtbyte *src,
				 unsigned_char_dynarr *dst,
				 int *code_out);

#endif /* INCLUDED_file_coding_h_ */

