Commit 56f4d68f787bd41bbd705a494c955d48784ace5f - golang-github-rivo-uniseg

+12

-12

grapheme.go less more

5	5	// user-perceived characters. While iterating, it also provides information
6	6	// about word boundaries, sentence boundaries, and line breaks.
7	7	//
8		// After constructing the class via NewGraphemes(str) for a given string "str",
9		// Next() is called for every grapheme cluster in a loop until it returns false.
	8	// After constructing the class via [NewGraphemes] for a given string "str",
	9	// [Next] is called for every grapheme cluster in a loop until it returns false.
10	10	// Inside the loop, information about the grapheme cluster as well as boundary
11	11	// information is available via the various methods (see examples below).
12	12	//
13	13	// Using this class to iterate over a string is convenient but it is much slower
14		// than using this package's Step() or StepString() functions or any of the
	14	// than using this package's [Step] or [StepString] functions or any of the
15	15	// other specialized functions starting with "First".
16	16	type Graphemes struct {
17	17	// The original string.

59	59	}
60	60
61	61	// Runes returns a slice of runes (code points) which corresponds to the current
62		// grapheme cluster. If the iterator is already past the end or Next() has not
	62	// grapheme cluster. If the iterator is already past the end or [Next] has not
63	63	// yet been called, nil is returned.
64	64	func (g *Graphemes) Runes() []rune {
65	65	if g.state < 0 {

69	69	}
70	70
71	71	// Str returns a substring of the original string which corresponds to the
72		// current grapheme cluster. If the iterator is already past the end or Next()
	72	// current grapheme cluster. If the iterator is already past the end or [Next]
73	73	// has not yet been called, an empty string is returned.
74	74	func (g *Graphemes) Str() string {
75	75	return g.cluster
76	76	}
77	77
78	78	// Bytes returns a byte slice which corresponds to the current grapheme cluster.
79		// If the iterator is already past the end or Next() has not yet been called,
	79	// If the iterator is already past the end or [Next] has not yet been called,
80	80	// nil is returned.
81	81	func (g *Graphemes) Bytes() []byte {
82	82	if g.state < 0 {

89	89	// positions into the original string. The first returned value "from" indexes
90	90	// the first byte and the second returned value "to" indexes the first byte that
91	91	// is not included anymore, i.e. str[from:to] is the current grapheme cluster of
92		// the original string "str". If Next() has not yet been called, both values are
	92	// the original string "str". If [Next] has not yet been called, both values are
93	93	// 0. If the iterator is already past the end, both values are 1.
94	94	func (g *Graphemes) Positions() (int, int) {
95	95	if g.state == -1 {

119	119	}
120	120
121	121	// LineBreak returns whether the line can be broken after the current grapheme
122		// cluster. A value of LineDontBreak means the line may not be broken, a value
123		// of LineMustBreak means the line must be broken, and a value of LineCanBreak
124		// means the line may or may not be broken.
	122	// cluster. A value of [LineDontBreak] means the line may not be broken, a value
	123	// of [LineMustBreak] means the line must be broken, and a value of
	124	// [LineCanBreak] means the line may or may not be broken.
125	125	func (g *Graphemes) LineBreak() int {
126	126	if g.state == -1 {
127	127	return LineDontBreak

133	133	}
134	134
135	135	// Reset puts the iterator into its initial state such that the next call to
136		// Next() sets it to the first grapheme cluster again.
	136	// [Next] sets it to the first grapheme cluster again.
137	137	func (g *Graphemes) Reset() {
138	138	g.state = -1
139	139	g.offset = 0

209	209	}
210	210	}
211	211
212		// FirstGraphemeClusterInString is like FirstGraphemeCluster() but its input and
	212	// FirstGraphemeClusterInString is like [FirstGraphemeCluster] but its input and
213	213	// outputs are strings.
214	214	func FirstGraphemeClusterInString(str string, state int) (cluster, rest string, reserved, newState int) {
215	215	// An empty string returns nothing.

+17

-1

line.go less more

36	36	//
37	37	// Note that in accordance with UAX #14 LB3, the final segment will end with
38	38	// "mustBreak" set to true. You can choose to ignore this by checking if the
39		// length of the "rest" slice is 0.
	39	// length of the "rest" slice is 0 and calling [HasTrailingLineBreak] or
	40	// [HasTrailingLineBreakInString] on the last rune.
40	41	//
41	42	// Note also that this algorithm may break within grapheme clusters. This is
42	43	// addressed in Section 8.2 Example 6 of UAX #14. To avoid this, you can use

110	111	}
111	112	}
112	113	}
	114
	115	// HasTrailingLineBreak returns true if the last rune in the given byte slice is
	116	// one of the hard line break code points as defined in LB4 and LB5 of UAX #14.
	117	func HasTrailingLineBreak(b []byte) bool {
	118	r, _ := utf8.DecodeLastRune(b)
	119	property, _ := propertyWithGenCat(lineBreakCodePoints, r)
	120	return property == lbBK \|\| property == lbCR \|\| property == lbLF \|\| property == lbNL
	121	}
	122
	123	// HasTrailingLineBreakInString is like [HasTrailingLineBreak] but for a string.
	124	func HasTrailingLineBreakInString(str string) bool {
	125	r, _ := utf8.DecodeLastRuneInString(str)
	126	property, _ := propertyWithGenCat(lineBreakCodePoints, r)
	127	return property == lbBK \|\| property == lbCR \|\| property == lbLF \|\| property == lbNL
	128	}

+1

-1

sentence.go less more

50	50	}
51	51	}
52	52
53		// FirstSentenceInString is like FirstSentence() but its input and outputs are
	53	// FirstSentenceInString is like [FirstSentence] but its input and outputs are
54	54	// strings.
55	55	func FirstSentenceInString(str string, state int) (sentence, rest string, newState int) {
56	56	// An empty byte slice returns nothing.

+3

-2

step.go less more

75	75	//
76	76	// Note that in accordance with UAX #14 LB3, the final segment will end with
77	77	// a mandatory line break (boundaries&MaskLine == LineMustBreak). You can choose
78		// to ignore this by checking if the length of the "rest" slice is 0.
	78	// to ignore this by checking if the length of the "rest" slice is 0 and calling
	79	// [HasTrailingLineBreak] or [HasTrailingLineBreakInString] on the last rune.
79	80	func Step(b []byte, state int) (cluster, rest []byte, boundaries int, newState int) {
80	81	// An empty byte slice returns nothing.
81	82	if len(b) == 0 {

135	136	}
136	137	}
137	138
138		// StepString is like Step() but its input and outputs are strings.
	139	// StepString is like [Step] but its input and outputs are strings.
139	140	func StepString(str string, state int) (cluster, rest string, boundaries int, newState int) {
140	141	// An empty byte slice returns nothing.
141	142	if len(str) == 0 {

+1

-1

word.go less more

50	50	}
51	51	}
52	52
53		// FirstWordInString is like FirstWord() but its input and outputs are strings.
	53	// FirstWordInString is like [FirstWord] but its input and outputs are strings.
54	54	func FirstWordInString(str string, state int) (word, rest string, newState int) {
55	55	// An empty byte slice returns nothing.
56	56	if len(str) == 0 {