fix(sanitizer): treat correct U+FFFD as normal character (#128)

yin1999 · web-flow · commit 3d89f8c2e0ca · 2023-08-11T12:36:45.000-07:00
diff --git a/pkg/asciisanitizer/sanitizer.go b/pkg/asciisanitizer/sanitizer.go
@@ -47,7 +47,7 @@ func (t *Sanitizer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
 			return
 		}
 		r, size := utf8.DecodeRune(src)
-		if r == utf8.RuneError {
+		if r == utf8.RuneError && size < 2 {
 			if !atEOF {
 				err = transform.ErrShortSrc
 				return

Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ func (t *Sanitizer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err`
`47`	`47`	`return`
`48`	`48`	`}`
`49`	`49`	`r, size := utf8.DecodeRune(src)`
`50`		`- if r == utf8.RuneError {`
	`50`	`+ if r == utf8.RuneError && size < 2 {`
`51`	`51`	`if !atEOF {`
`52`	`52`	`err = transform.ErrShortSrc`
`53`	`53`	`return`