mbyte.c File Reference

#include <inttypes.h>
#include <stdbool.h>
#include <string.h>
#include <wchar.h>
#include <wctype.h>
#include "nvim/ascii.h"
#include "nvim/vim.h"
#include "nvim/arabic.h"
#include "nvim/charset.h"
#include "nvim/cursor.h"
#include "nvim/eval.h"
#include "nvim/fileio.h"
#include "nvim/func_attr.h"
#include "nvim/iconv.h"
#include "nvim/mark.h"
#include "nvim/mbyte.h"
#include "nvim/memline.h"
#include "nvim/memory.h"
#include "nvim/message.h"
#include "nvim/misc1.h"
#include "nvim/option.h"
#include "nvim/os/os.h"
#include "nvim/path.h"
#include "nvim/screen.h"
#include "nvim/spell.h"
#include "nvim/strings.h"

Data Structures
struct	convertStruct
struct	interval

Macros
#define	IDX_LATIN_1   0
#define	IDX_ISO_2   1
#define	IDX_ISO_3   2
#define	IDX_ISO_4   3
#define	IDX_ISO_5   4
#define	IDX_ISO_6   5
#define	IDX_ISO_7   6
#define	IDX_ISO_8   7
#define	IDX_ISO_9   8
#define	IDX_ISO_10   9
#define	IDX_ISO_11   10
#define	IDX_ISO_13   11
#define	IDX_ISO_14   12
#define	IDX_ISO_15   13
#define	IDX_KOI8_R   14
#define	IDX_KOI8_U   15
#define	IDX_UTF8   16
#define	IDX_UCS2   17
#define	IDX_UCS2LE   18
#define	IDX_UTF16   19
#define	IDX_UTF16LE   20
#define	IDX_UCS4   21
#define	IDX_UCS4LE   22
#define	IDX_DEBUG   23
#define	IDX_EUC_JP   24
#define	IDX_SJIS   25
#define	IDX_EUC_KR   26
#define	IDX_EUC_CN   27
#define	IDX_EUC_TW   28
#define	IDX_BIG5   29
#define	IDX_CP437   30
#define	IDX_CP737   31
#define	IDX_CP775   32
#define	IDX_CP850   33
#define	IDX_CP852   34
#define	IDX_CP855   35
#define	IDX_CP857   36
#define	IDX_CP860   37
#define	IDX_CP861   38
#define	IDX_CP862   39
#define	IDX_CP863   40
#define	IDX_CP865   41
#define	IDX_CP866   42
#define	IDX_CP869   43
#define	IDX_CP874   44
#define	IDX_CP932   45
#define	IDX_CP936   46
#define	IDX_CP949   47
#define	IDX_CP950   48
#define	IDX_CP1250   49
#define	IDX_CP1251   50
#define	IDX_CP1253   51
#define	IDX_CP1254   52
#define	IDX_CP1255   53
#define	IDX_CP1256   54
#define	IDX_CP1257   55
#define	IDX_CP1258   56
#define	IDX_MACROMAN   57
#define	IDX_HPROMAN8   58
#define	IDX_COUNT   59

Functions
int	enc_canon_props (const char_u *name)
int	bomb_size (void)
void	remove_bom (char_u *s)
int	mb_get_class (const char_u *p)
int	mb_get_class_tab (const char_u *p, const uint64_t *const chartab)
int	utf_char2cells (int c)
int	utf_ptr2cells (const char_u *p)
int	utf_ptr2cells_len (const char_u *p, int size)
size_t	mb_string2cells (const char_u *str)
size_t	mb_string2cells_len (const char_u *str, size_t size) FUNC_ATTR_NONNULL_ARG(1)
int	utf_ptr2char (const char_u *const p) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT
int	mb_ptr2char_adv (const char_u **const pp)
int	mb_cptr2char_adv (const char_u **pp)
bool	utf_composinglike (const char_u *p1, const char_u *p2)
int	utfc_ptr2char (const char_u *p, int *pcc)
int	utfc_ptr2char_len (const char_u *p, int *pcc, int maxlen)
int	utf_ptr2len (const char_u *const p) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
int	utf_byte2len (int b)
int	utf_ptr2len_len (const char_u *p, int size)
int	utfc_ptr2len (const char_u *const p) FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
int	utfc_ptr2len_len (const char_u *p, int size)
int	utf_char2len (const int c)
	Determine how many bytes certain unicode codepoint will occupy. More...
int	utf_char2bytes (const int c, char_u *const buf)
bool	utf_iscomposing (int c)
bool	utf_printable (int c)
int	utf_class (const int c)
int	utf_class_tab (const int c, const uint64_t *const chartab)
bool	utf_ambiguous_width (int c)
int	utf_fold (int a)
int	mb_toupper (int a)
bool	mb_islower (int a)
int	mb_tolower (int a)
bool	mb_isupper (int a)
void	mb_utflen (const char_u *s, size_t len, size_t *codepoints, size_t *codeunits) FUNC_ATTR_NONNULL_ALL
ssize_t	mb_utf_index_to_bytes (const char_u *s, size_t len, size_t index, bool use_utf16_units) FUNC_ATTR_NONNULL_ALL
int	mb_strnicmp (const char_u *s1, const char_u *s2, const size_t nn)
int	mb_stricmp (const char *s1, const char *s2)
void	show_utf8 (void)
int	utf_head_off (const char_u *base, const char_u *p)
bool	utf_eat_space (int cc) FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT
bool	utf_allow_break_before (int cc) FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT
bool	utf_allow_break_after (int cc) FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT
bool	utf_allow_break (int cc, int ncc) FUNC_ATTR_CONST FUNC_ATTR_WARN_UNUSED_RESULT
void	mb_copy_char (const char_u **const fp, char_u **const tp)
int	mb_off_next (char_u *base, char_u *p)
int	mb_tail_off (char_u *base, char_u *p)
int	mb_head_off (char_u *base, char_u *p)
void	utf_find_illegal (void)
void	mb_adjust_cursor (void)
void	mb_check_adjust_col (void *win_)
char_u *	mb_prevptr (char_u *line, char_u *p)
int	mb_charlen (char_u *str)
int	mb_charlen_len (char_u *str, int len)
const char *	mb_unescape (const char **const pp) FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
char_u *	enc_skip (char_u *p)
char_u *	enc_canonize (char_u *enc) FUNC_ATTR_NONNULL_RET
char_u *	enc_locale (void)
int	convert_setup (vimconv_T *vcp, char_u *from, char_u *to)
int	convert_setup_ext (vimconv_T *vcp, char_u *from, bool from_unicode_is_utf8, char_u *to, bool to_unicode_is_utf8)
char_u *	string_convert (const vimconv_T *const vcp, char_u *ptr, size_t *lenp)
char_u *	string_convert_ext (const vimconv_T *const vcp, char_u *ptr, size_t *lenp, size_t *unconvlenp)

Variables
const uint8_t	utf8len_tab []
const uint8_t	utf8len_tab_zero []

Macro Definition Documentation

IDX_BIG5

#define IDX_BIG5   29

IDX_COUNT

#define IDX_COUNT   59

IDX_CP1250

#define IDX_CP1250   49

IDX_CP1251

#define IDX_CP1251   50

IDX_CP1253

#define IDX_CP1253   51

IDX_CP1254

#define IDX_CP1254   52

IDX_CP1255

#define IDX_CP1255   53

IDX_CP1256

#define IDX_CP1256   54

IDX_CP1257

#define IDX_CP1257   55

IDX_CP1258

#define IDX_CP1258   56

IDX_CP437

#define IDX_CP437   30

IDX_CP737

#define IDX_CP737   31

IDX_CP775

#define IDX_CP775   32

IDX_CP850

#define IDX_CP850   33

IDX_CP852

#define IDX_CP852   34

IDX_CP855

#define IDX_CP855   35

IDX_CP857

#define IDX_CP857   36

IDX_CP860

#define IDX_CP860   37

IDX_CP861

#define IDX_CP861   38

IDX_CP862

#define IDX_CP862   39

IDX_CP863

#define IDX_CP863   40

IDX_CP865

#define IDX_CP865   41

IDX_CP866

#define IDX_CP866   42

IDX_CP869

#define IDX_CP869   43

IDX_CP874

#define IDX_CP874   44

IDX_CP932

#define IDX_CP932   45

IDX_CP936

#define IDX_CP936   46

IDX_CP949

#define IDX_CP949   47

IDX_CP950

#define IDX_CP950   48

IDX_DEBUG

#define IDX_DEBUG   23

IDX_EUC_CN

#define IDX_EUC_CN   27

IDX_EUC_JP

#define IDX_EUC_JP   24

IDX_EUC_KR

#define IDX_EUC_KR   26

IDX_EUC_TW

#define IDX_EUC_TW   28

IDX_HPROMAN8

#define IDX_HPROMAN8   58

IDX_ISO_10

#define IDX_ISO_10   9

IDX_ISO_11

#define IDX_ISO_11   10

IDX_ISO_13

#define IDX_ISO_13   11

IDX_ISO_14

#define IDX_ISO_14   12

IDX_ISO_15

#define IDX_ISO_15   13

IDX_ISO_2

#define IDX_ISO_2   1

IDX_ISO_3

#define IDX_ISO_3   2

IDX_ISO_4

#define IDX_ISO_4   3

IDX_ISO_5

#define IDX_ISO_5   4

IDX_ISO_6

#define IDX_ISO_6   5

IDX_ISO_7

#define IDX_ISO_7   6

IDX_ISO_8

#define IDX_ISO_8   7

IDX_ISO_9

#define IDX_ISO_9   8

IDX_KOI8_R

#define IDX_KOI8_R   14

IDX_KOI8_U

#define IDX_KOI8_U   15

IDX_LATIN_1

#define IDX_LATIN_1   0

IDX_MACROMAN

#define IDX_MACROMAN   57

IDX_SJIS

#define IDX_SJIS   25

IDX_UCS2

#define IDX_UCS2   17

IDX_UCS2LE

#define IDX_UCS2LE   18

IDX_UCS4

#define IDX_UCS4   21

IDX_UCS4LE

#define IDX_UCS4LE   22

IDX_UTF16

#define IDX_UTF16   19

IDX_UTF16LE

#define IDX_UTF16LE   20

IDX_UTF8

#define IDX_UTF8   16

Function Documentation

bomb_size()

int bomb_size

(

void

)

convert_setup()

int convert_setup	(	vimconv_T *	vcp,
		char_u *	from,
		char_u *	to
	)

convert_setup_ext()

int convert_setup_ext	(	vimconv_T *	vcp,
		char_u *	from,
		bool	from_unicode_is_utf8,
		char_u *	to,
		bool	to_unicode_is_utf8
	)

As convert_setup(), but only when from_unicode_is_utf8 is true will all "from" unicode charsets be considered utf-8. Same for "to".

enc_canon_props()

int enc_canon_props

(

const char_u *

name

)

enc_canonize()

char_u* enc_canonize

(

char_u *

enc

)

enc_locale()

char_u* enc_locale

(

void

)

enc_skip()

char_u* enc_skip

(

char_u *

p

)

mb_adjust_cursor()

void mb_adjust_cursor

(

void

)

mb_charlen()

int mb_charlen

(

char_u *

str

)

mb_charlen_len()

int mb_charlen_len	(	char_u *	str,
		int	len
	)

mb_check_adjust_col()

void mb_check_adjust_col

(

void *

win_

)

Checks and adjusts cursor column. Not mode-dependent.

See also

check_cursor_col_win

Parameters

win_	Places cursor on a valid column for this window.

mb_copy_char()

void mb_copy_char	(	const char_u **const	fp,
		char_u **const	tp
	)

Copy a character, advancing the pointers

Parameters

[in,out]	fp	Source of the character to copy.
[in,out]	tp	Destination to copy to.

mb_cptr2char_adv()

int mb_cptr2char_adv

(

const char_u **

pp

)

mb_get_class()

int mb_get_class

(

const char_u *

p

)

mb_get_class_tab()

int mb_get_class_tab	(	const char_u *	p,
		const uint64_t *const	chartab
	)

mb_head_off()

int mb_head_off	(	char_u *	base,
		char_u *	p
	)

Return the offset from "p" to the first byte of the character it points into. Can start anywhere in a stream of bytes.

Parameters

[in]	base	Pointer to start of string
[in]	p	Pointer to byte for which to return the offset to the previous codepoint

Returns

0 if invalid sequence, else offset to previous codepoint

mb_islower()

bool mb_islower

(

int

a

)

mb_isupper()

bool mb_isupper

(

int

a

)

mb_off_next()

int mb_off_next	(	char_u *	base,
		char_u *	p
	)

mb_prevptr()

char_u* mb_prevptr	(	char_u *	line,
		char_u *	p
	)

Parameters

line	start of the string

Returns

a pointer to the character before "*p", if there is one.

mb_ptr2char_adv()

int mb_ptr2char_adv

(

const char_u **const

pp

)

mb_stricmp()

int mb_stricmp	(	const char *	s1,
		const char *	s2
	)

Compare strings case-insensitively

Note

We need to call mb_stricmp() even when we aren't dealing with a multi-byte encoding because mb_stricmp() takes care of all ASCII and non-ascii encodings, including characters with umlauts in latin1, etc., while STRICMP() only handles the system locale version, which often does not handle non-ascii properly.

Parameters

[in]	s1	First string to compare, not more then MAXCOL characters.
[in]	s2	Second string to compare, not more then MAXCOL characters.

Returns

0 if strings are equal, <0 if s1 < s2, >0 if s1 > s2.

mb_string2cells()

size_t mb_string2cells

(

const char_u *

str

)

Calculate the number of cells occupied by string str.

Parameters

str	The source string, may not be NULL, must be a NUL-terminated string.

Returns

The number of cells occupied by string str

mb_string2cells_len()

size_t mb_string2cells_len	(	const char_u *	str,
		size_t	size
	)

Get the number of cells occupied by string str with maximum length size

Parameters

str	The source string, may not be NULL, must be a NUL-terminated string.
size	maximum length of string. It will terminate on earlier NUL.

Returns

The number of cells occupied by string str

mb_strnicmp()

int mb_strnicmp	(	const char_u *	s1,
		const char_u *	s2,
		const size_t	nn
	)

mb_tail_off()

int mb_tail_off	(	char_u *	base,
		char_u *	p
	)

mb_tolower()

int mb_tolower

(

int

a

)

Return the lower-case equivalent of "a", which is a UCS-4 character. Use simple case folding.

mb_toupper()

int mb_toupper

(

int

a

)

Return the upper-case equivalent of "a", which is a UCS-4 character. Use simple case folding.

mb_unescape()

const char* mb_unescape

(

const char **const

pp

)

Try to unescape a multibyte character

Used for the rhs and lhs of the mappings.

Parameters

[in,out]

pp

String to unescape. Is advanced to just after the bytes that form a multibyte character.

Returns

Unescaped string if it is a multibyte character, NULL if no multibyte character was found. Returns a static buffer, always one and the same.

mb_utf_index_to_bytes()

ssize_t mb_utf_index_to_bytes	(	const char_u *	s,
		size_t	len,
		size_t	index,
		bool	use_utf16_units
	)

mb_utflen()

void mb_utflen	(	const char_u *	s,
		size_t	len,
		size_t *	codepoints,
		size_t *	codeunits
	)

Measure the length of a string in corresponding UTF-32 and UTF-16 units.

Invalid UTF-8 bytes, or embedded surrogates, count as one code point/unit each.

The out parameters are incremented. This is used to measure the size of a buffer region consisting of multiple line segments.

Parameters

	s	the string
	len	maximum length (an earlier NUL terminates)
[out]	codepoints	incremented with UTF-32 code point size
[out]	codeunits	incremented with UTF-16 code unit size

remove_bom()

void remove_bom

(

char_u *

s

)

show_utf8()

void show_utf8

(

void

)

string_convert()

char_u* string_convert	(	const vimconv_T *const	vcp,
		char_u *	ptr,
		size_t *	lenp
	)

string_convert_ext()

char_u* string_convert_ext	(	const vimconv_T *const	vcp,
		char_u *	ptr,
		size_t *	lenp,
		size_t *	unconvlenp
	)

utf_allow_break()

bool utf_allow_break	(	int	cc,
		int	ncc
	)

utf_allow_break_after()

bool utf_allow_break_after

(

int

cc

)

utf_allow_break_before()

bool utf_allow_break_before

(

int

cc

)

utf_ambiguous_width()

bool utf_ambiguous_width

(

int

c

)

utf_byte2len()

int utf_byte2len

(

int

b

)

utf_char2bytes()

int utf_char2bytes	(	const int	c,
		char_u *const	buf
	)

Convert Unicode character to UTF-8 string

Parameters

	c	character to convert to buf
[out]	buf	UTF-8 string generated from c, does not add \0

Returns

Number of bytes (1-6).

utf_char2cells()

int utf_char2cells

(

int

c

)

For UTF-8 character "c" return 2 for a double-width character, 1 for others. Returns 4 or 6 for an unprintable character. Is only correct for characters >= 0x80. When p_ambw is "double", return 2 for a character with East Asian Width class 'A'(mbiguous).

Note

Tables doublewidth and ambiguous are generated by gen_unicode_tables.lua, which must be manually invoked as needed.

utf_char2len()

int utf_char2len

(

const int

c

)

Determine how many bytes certain unicode codepoint will occupy.

utf_class()

int utf_class

(

const int

c

)

utf_class_tab()

int utf_class_tab	(	const int	c,
		const uint64_t *const	chartab
	)

utf_composinglike()

bool utf_composinglike	(	const char_u *	p1,
		const char_u *	p2
	)

utf_eat_space()

bool utf_eat_space

(

int

cc

)

utf_find_illegal()

void utf_find_illegal

(

void

)

utf_fold()

int utf_fold

(

int

a

)

utf_head_off()

int utf_head_off	(	const char_u *	base,
		const char_u *	p
	)

Return offset from "p" to the first byte of the character it points into. If "p" points to the NUL at the end of the string return 0. Returns 0 when already at the first byte of a character.

utf_iscomposing()

bool utf_iscomposing

(

int

c

)

utf_printable()

bool utf_printable

(

int

c

)

utf_ptr2cells()

int utf_ptr2cells

(

const char_u *

p

)

Return the number of display cells character at "*p" occupies. This doesn't take care of unprintable characters, use ptr2cells() for that.

utf_ptr2cells_len()

int utf_ptr2cells_len	(	const char_u *	p,
		int	size
	)

Like utf_ptr2cells(), but limit string length to "size". For an empty string or truncated character returns 1.

utf_ptr2char()

int utf_ptr2char

(

const char_u *const

p

)

Convert a UTF-8 byte sequence to a character number.

If the sequence is illegal or truncated by a NUL then the first byte is returned. For an overlong sequence this may return zero. Does not include composing characters for obvious reasons.

Parameters

[in]

p

String to convert.

Returns

Unicode codepoint or byte value.

utf_ptr2len()

int utf_ptr2len

(

const char_u *const

p

)

Get the length of a UTF-8 byte sequence representing a single codepoint

Parameters

[in]

p

UTF-8 string.

Returns

Sequence length, 0 for empty string and 1 for non-UTF-8 byte sequence.

utf_ptr2len_len()

int utf_ptr2len_len	(	const char_u *	p,
		int	size
	)

utfc_ptr2char()

int utfc_ptr2char	(	const char_u *	p,
		int *	pcc
	)

Convert a UTF-8 string to a wide character

Also gets up to MAX_MCO composing characters.

Parameters

[out]

pcc

Location where to store composing characters. Must have space at least for MAX_MCO + 1 elements.

Returns

leading character.

utfc_ptr2char_len()

int utfc_ptr2char_len	(	const char_u *	p,
		int *	pcc,
		int	maxlen
	)

utfc_ptr2len()

int utfc_ptr2len

(

const char_u *const

p

)

Return the number of bytes occupied by a UTF-8 character in a string

This includes following composing characters.

utfc_ptr2len_len()

int utfc_ptr2len_len	(	const char_u *	p,
		int	size
	)

Variable Documentation

canon

int canon

codepage

int codepage

name

const char* name

prop

int prop

utf8len_tab

const uint8_t utf8len_tab[]

Initial value:

= {
  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  
  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,  
  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1,  
}

utf8len_tab_zero

const uint8_t utf8len_tab_zero[]

Initial value:

= {
  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,  
  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,  
  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0,  
}