charset

package
v0.0.0-...-503c688 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 8, 2023 License: Apache-2.0, Apache-2.0 Imports: 21 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// CollationBin is the default collation for CharsetBin.
	CollationBin = "binary"
	// CollationUTF8 is the default collation for CharsetUTF8.
	CollationUTF8 = "utf8_bin"
	// CollationUTF8MB4 is the default collation for CharsetUTF8MB4.
	CollationUTF8MB4 = "utf8mb4_bin"
	// CollationASCII is the default collation for CharsetACSII.
	CollationASCII = "ascii_bin"
	// CollationLatin1 is the default collation for CharsetLatin1.
	CollationLatin1 = "latin1_bin"
	// CollationGBKBin is the default collation for CharsetGBK when new collation is disabled.
	CollationGBKBin = "gbk_bin"
	// CollationGBKChineseCI is the default collation for CharsetGBK when new collation is enabled.
	CollationGBKChineseCI = "gbk_chinese_ci"
)
View Source
const (
	// CharsetASCII is a subset of UTF8.
	CharsetASCII = "ascii"
	// CharsetBin is used for marking binary charset.
	CharsetBin = "binary"
	// CharsetLatin1 is a single byte charset.
	CharsetLatin1 = "latin1"
	// CharsetUTF8 is the default charset for string types.
	CharsetUTF8 = "utf8"
	// CharsetUTF8MB4 represents 4 bytes utf8, which works the same way as utf8 in Go.
	CharsetUTF8MB4 = "utf8mb4"
	//revive:disable:exported
	CharsetARMSCII8 = "armscii8"
	CharsetBig5     = "big5"
	CharsetCP1250   = "cp1250"
	CharsetCP1251   = "cp1251"
	CharsetCP1256   = "cp1256"
	CharsetCP1257   = "cp1257"
	CharsetCP850    = "cp850"
	CharsetCP852    = "cp852"
	CharsetCP866    = "cp866"
	CharsetCP932    = "cp932"
	CharsetDEC8     = "dec8"
	CharsetEUCJPMS  = "eucjpms"
	CharsetEUCKR    = "euckr"
	CharsetGB18030  = "gb18030"
	CharsetGB2312   = "gb2312"
	CharsetGBK      = "gbk"
	CharsetGEOSTD8  = "geostd8"
	CharsetGreek    = "greek"
	CharsetHebrew   = "hebrew"
	CharsetHP8      = "hp8"
	CharsetKEYBCS2  = "keybcs2"
	CharsetKOI8R    = "koi8r"
	CharsetKOI8U    = "koi8u"
	CharsetLatin2   = "latin2"
	CharsetLatin5   = "latin5"
	CharsetLatin7   = "latin7"
	CharsetMacCE    = "macce"
	CharsetMacRoman = "macroman"
	CharsetSJIS     = "sjis"
	CharsetSWE7     = "swe7"
	CharsetTIS620   = "tis620"
	CharsetUCS2     = "ucs2"
	CharsetUJIS     = "ujis"
	CharsetUTF16    = "utf16"
	CharsetUTF16LE  = "utf16le"
	CharsetUTF32    = "utf32"
)
View Source
const (
	// OpReplaceNoErr is used to replace invalid bytes with '?'.
	OpReplaceNoErr  = opFromUTF8 | opTruncateReplace | opCollectFrom | opSkipError
	OpReplace       = opFromUTF8 | opTruncateReplace | opCollectFrom
	OpEncode        = opFromUTF8 | opTruncateTrim | opCollectTo
	OpEncodeNoErr   = OpEncode | opSkipError
	OpEncodeReplace = opFromUTF8 | opTruncateReplace | opCollectTo
	OpDecode        = opToUTF8 | opTruncateTrim | opCollectTo
	OpDecodeNoErr   = OpDecode | opSkipError
	OpDecodeReplace = opToUTF8 | opTruncateReplace | opCollectTo
)

Variables

View Source
var (
	// ErrUnknownCollation is unknown collation.
	ErrUnknownCollation = terror.ClassDDL.NewStd(mysql.ErrUnknownCollation)
	// ErrCollationCharsetMismatch is collation charset mismatch.
	ErrCollationCharsetMismatch = terror.ClassDDL.NewStd(mysql.ErrCollationCharsetMismatch)
)
View Source
var CharacterSetInfos = map[string]*Charset{
	CharsetUTF8:    {CharsetUTF8, CollationUTF8, make(map[string]*Collation), "UTF-8 Unicode", 3},
	CharsetUTF8MB4: {CharsetUTF8MB4, CollationUTF8MB4, make(map[string]*Collation), "UTF-8 Unicode", 4},
	CharsetASCII:   {CharsetASCII, CollationASCII, make(map[string]*Collation), "US ASCII", 1},
	CharsetLatin1:  {CharsetLatin1, CollationLatin1, make(map[string]*Collation), "Latin1", 1},
	CharsetBin:     {CharsetBin, CollationBin, make(map[string]*Collation), "binary", 1},
	CharsetGBK:     {CharsetGBK, CollationGBKBin, make(map[string]*Collation), "Chinese Internal Code Specification", 2},
}

CharacterSetInfos contains all the supported charsets.

View Source
var EncodingASCIIImpl = &encodingASCII{encodingBase{enc: encoding.Nop}}

EncodingASCIIImpl is the instance of encodingASCII

View Source
var EncodingBinImpl = &encodingBin{encodingBase{enc: encoding.Nop}}

EncodingBinImpl is the instance of encodingBin.

View Source
var EncodingGBKImpl = &encodingGBK{encodingBase{enc: customGBK{}}}

EncodingGBKImpl is the instance of encodingGBK

View Source
var EncodingLatin1Impl = &encodingLatin1{encodingUTF8{encodingBase{enc: encoding.Nop}}}

EncodingLatin1Impl is the instance of encodingLatin1. TiDB uses utf8 implementation for latin1 charset because of the backward compatibility.

View Source
var EncodingUTF8Impl = &encodingUTF8{encodingBase{enc: encoding.Nop}}

EncodingUTF8Impl is the instance of encodingUTF8.

View Source
var EncodingUTF8MB3StrictImpl = &encodingUTF8MB3Strict{
	encodingUTF8{
		encodingBase{
			enc: encoding.Nop,
		},
	},
}

EncodingUTF8MB3StrictImpl is the instance of encodingUTF8MB3Strict.

View Source
var ErrInvalidCharacterString = terror.ClassParser.NewStd(mysql.ErrInvalidCharacterString)

ErrInvalidCharacterString returns when the string is invalid in the specific charset.

View Source
var GBKCase = unicode.SpecialCase{
	unicode.CaseRange{Lo: 0x00E0, Hi: 0x00E1, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x00E8, Hi: 0x00EA, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x00EC, Hi: 0x00ED, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x00F2, Hi: 0x00F3, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x00F9, Hi: 0x00FA, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x00FC, Hi: 0x00FC, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x0101, Hi: 0x0101, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x0113, Hi: 0x0113, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x011B, Hi: 0x011B, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x012B, Hi: 0x012B, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x0144, Hi: 0x0144, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x0148, Hi: 0x0148, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x014D, Hi: 0x014D, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x016B, Hi: 0x016B, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01CE, Hi: 0x01CE, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01D0, Hi: 0x01D0, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01D2, Hi: 0x01D2, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01D4, Hi: 0x01D4, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01D6, Hi: 0x01D6, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01D8, Hi: 0x01D8, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01DA, Hi: 0x01DA, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x01DC, Hi: 0x01DC, Delta: [unicode.MaxCase]rune{0, 0, 0}},
	unicode.CaseRange{Lo: 0x216A, Hi: 0x216B, Delta: [unicode.MaxCase]rune{0, 0, 0}},
}

GBKCase follows https://dev.mysql.com/worklog/task/?id=4583.

View Source
var TiFlashSupportedCharsets = map[string]struct{}{
	CharsetUTF8:    {},
	CharsetUTF8MB4: {},
	CharsetASCII:   {},
	CharsetLatin1:  {},
	CharsetBin:     {},
}

TiFlashSupportedCharsets is a map which contains TiFlash supports charsets.

Functions

func AddCharset

func AddCharset(c *Charset)

AddCharset adds a new charset. Use only when adding a custom charset to the parser.

func AddCollation

func AddCollation(c *Collation)

AddCollation adds a new collation. Use only when adding a custom collation to the parser.

func AddSupportedCollation

func AddSupportedCollation(c *Collation)

AddSupportedCollation adds a new collation into supportedCollations. Use only when adding a custom collation to the parser.

func CountValidBytes

func CountValidBytes(e Encoding, src []byte) int

CountValidBytes counts the first valid bytes in src that can be encoded to the current encoding.

func CountValidBytesDecode

func CountValidBytesDecode(e Encoding, src []byte) int

CountValidBytesDecode counts the first valid bytes in src that can be decoded to utf-8.

func GetCharsetInfoByID

func GetCharsetInfoByID(coID int) (charsetStr string, collateStr string, err error)

GetCharsetInfoByID returns charset and collation for id as cs_number.

func GetDefaultCharsetAndCollate

func GetDefaultCharsetAndCollate() (defaultCharset string, defaultCollationName string)

GetDefaultCharsetAndCollate returns the default charset and collation.

func GetDefaultCollation

func GetDefaultCollation(charset string) (string, error)

GetDefaultCollation returns the default collation for charset.

func GetDefaultCollationLegacy

func GetDefaultCollationLegacy(charset string) (string, error)

GetDefaultCollationLegacy is compatible with the charset support in old version parser.

func HackSlice

func HackSlice(s string) (b []byte)

HackSlice converts string to slice without copy. Use at your own risk.

func HackString

func HackString(b []byte) (s string)

HackString converts slice to string without copy. Use it at your own risk.

func IsSupportedEncoding

func IsSupportedEncoding(charset string) bool

IsSupportedEncoding checks if the charset is fully supported.

func Lookup

func Lookup(label string) (e encoding.Encoding, name string)

Lookup returns the encoding with the specified label, and its canonical name. It returns nil and the empty string if label is not one of the standard encodings for HTML. Matching is case-insensitive and ignores leading and trailing whitespace.

func NewCustomGBKEncoder

func NewCustomGBKEncoder() *encoding.Encoder

NewCustomGBKEncoder return a custom GBK encoding.

func RemoveCharset

func RemoveCharset(c string)

RemoveCharset remove a charset. Use only when remove a custom charset to the parser.

func ValidCharsetAndCollation

func ValidCharsetAndCollation(cs string, co string) bool

ValidCharsetAndCollation checks the charset and the collation validity and returns a boolean.

Types

type Charset

type Charset struct {
	Name             string
	DefaultCollation string
	Collations       map[string]*Collation
	Desc             string
	Maxlen           int
}

Charset is a charset. Now we only support MySQL.

func GetCharsetInfo

func GetCharsetInfo(cs string) (*Charset, error)

GetCharsetInfo returns charset and collation for cs as name.

func GetSupportedCharsets

func GetSupportedCharsets() []*Charset

GetSupportedCharsets gets descriptions for all charsets supported so far.

type Collation

type Collation struct {
	ID          int
	CharsetName string
	Name        string
	IsDefault   bool
}

Collation is a collation. Now we only support MySQL.

func GetCollationByID

func GetCollationByID(id int) (*Collation, error)

GetCollationByID returns collations by given id.

func GetCollationByName

func GetCollationByName(name string) (*Collation, error)

GetCollationByName returns the collation by name.

func GetCollations

func GetCollations() []*Collation

GetCollations returns a list for all collations.

func GetSupportedCollations

func GetSupportedCollations() []*Collation

GetSupportedCollations gets information for all collations supported so far.

type Encoding

type Encoding interface {
	// Name is the name of the encoding.
	Name() string
	// Tp is the type of the encoding.
	Tp() EncodingTp
	// Peek returns the next char.
	Peek(src []byte) []byte
	// MbLen returns multiple byte length, if the next character is single byte, return 0.
	MbLen(string) int
	// IsValid checks whether the utf-8 bytes can be convert to valid string in current encoding.
	IsValid(src []byte) bool
	// Foreach iterates the characters in in current encoding.
	Foreach(src []byte, op Op, fn func(from, to []byte, ok bool) bool)
	// Transform map the bytes in src to dest according to Op.
	// **the caller should initialize the dest if it wants to avoid memory alloc every time,
	//   or else it will always make a new one**
	//
	// **the returned array may be the alias of `src`, edit the returned array on your own risk**
	Transform(dest *bytes.Buffer, src []byte, op Op) ([]byte, error)
	// ToUpper change a string to uppercase.
	ToUpper(src string) string
	// ToLower change a string to lowercase.
	ToLower(src string) string
}

Encoding provide encode/decode functions for a string with a specific charset.

func FindEncoding

func FindEncoding(charset string) Encoding

FindEncoding finds the encoding according to charset.

func FindEncodingTakeUTF8AsNoop

func FindEncodingTakeUTF8AsNoop(charset string) Encoding

FindEncodingTakeUTF8AsNoop finds the encoding according to the charset except that utf-8 is treated as no-operation encoding. This is used to reduce the overhead of utf-8 validation in some cases.

type EncodingTp

type EncodingTp int8

EncodingTp is the type of the encoding.

const (
	EncodingTpNone EncodingTp = iota
	EncodingTpUTF8
	EncodingTpUTF8MB3Strict
	EncodingTpASCII
	EncodingTpLatin1
	EncodingTpBin
	EncodingTpGBK
)

type Op

type Op int16

Op is used by Encoding.Transform.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL