Fix bug when viewing the commit diff page with non-ANSI files (#36149)

Fix #35504

---------

Co-authored-by: wxiaoguang <[email protected]>

Author: lunny

modules/charset/charset.go

@@ -5,12 +5,10 @@ package charset
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"strings"
 	"unicode/utf8"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -23,135 +21,125 @@ import (
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
 type ConvertOpts struct {
-	KeepBOM bool
+	KeepBOM           bool
+	ErrorReplacement  []byte
+	ErrorReturnOrigin bool
 }
 
+var ToUTF8WithFallbackReaderPrefetchSize = 16 * 1024
+
 // ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
 func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
-	buf := make([]byte, 2048)
+	buf := make([]byte, ToUTF8WithFallbackReaderPrefetchSize)
 	n, err := util.ReadAtMost(rd, buf)
 	if err != nil {
-		return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
+		// read error occurs, don't do any processing
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
 	}
 
-	charsetLabel, err := DetectEncoding(buf[:n])
-	if err != nil || charsetLabel == "UTF-8" {
-		return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
+	charsetLabel, _ := DetectEncoding(buf[:n])
+	if charsetLabel == "UTF-8" {
+		// is utf-8, try to remove BOM and read it as-is
+		return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd)
 	}
 
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
+		// unknown charset, don't do any processing
 		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
 	}
 
+	// convert from charset to utf-8
 	return transform.NewReader(
-		io.MultiReader(
-			bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
-			rd,
-		),
+		io.MultiReader(bytes.NewReader(buf[:n]), rd),
 		encoding.NewDecoder(),
 	)
 }
 
-// ToUTF8 converts content to UTF8 encoding
-func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil {
-		return "", err
-	} else if charsetLabel == "UTF-8" {
-		return string(MaybeRemoveBOM(content, opts)), nil
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose much data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		result = append(result, content[n:]...)
-	}
-
-	result = MaybeRemoveBOM(result, opts)
-
-	return string(result), err
-}
-
 // ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
 	bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content), opts))
 	return bs
 }
 
-// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
-func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return MaybeRemoveBOM(content, opts)
+func ToUTF8DropErrors(content []byte) []byte {
+	return ToUTF8(content, ConvertOpts{ErrorReplacement: []byte{' '}})
+}
+
+func ToUTF8(content []byte, opts ConvertOpts) []byte {
+	charsetLabel, _ := DetectEncoding(content)
+	if charsetLabel == "UTF-8" {
+		return maybeRemoveBOM(content, opts)
 	}
 
 	encoding, _ := charset.Lookup(charsetLabel)
 	if encoding == nil {
+		setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel)
 		return content
 	}
 
-	// We ignore any non-decodable parts from the file.
-	// Some parts might be lost
 	var decoded []byte
 	decoder := encoding.NewDecoder()
 	idx := 0
-	for {
+	for idx < len(content) {
 		result, n, err := transform.Bytes(decoder, content[idx:])
 		decoded = append(decoded, result...)
 		if err == nil {
 			break
 		}
-		decoded = append(decoded, ' ')
-		idx = idx + n + 1
-		if idx >= len(content) {
-			break
+		if opts.ErrorReturnOrigin {
+			return content
+		}
+		if opts.ErrorReplacement == nil {
+			decoded = append(decoded, content[idx+n])
+		} else {
+			decoded = append(decoded, opts.ErrorReplacement...)
 		}
+		idx += n + 1
 	}
-
-	return MaybeRemoveBOM(decoded, opts)
+	return maybeRemoveBOM(decoded, opts)
 }
 
-// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
-func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
+// maybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
+func maybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
 	if opts.KeepBOM {
 		return content
 	}
-	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
-		return content[3:]
-	}
-	return content
+	return bytes.TrimPrefix(content, UTF8BOM)
 }
 
 // DetectEncoding detect the encoding of content
-func DetectEncoding(content []byte) (string, error) {
+// it always returns a detected or guessed "encoding" string, no matter error happens or not
+func DetectEncoding(content []byte) (encoding string, _ error) {
 	// First we check if the content represents valid utf8 content excepting a truncated character at the end.
 
 	// Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do
-	// instead we walk backwards from the end to trim off a the incomplete character
+	// instead we walk backwards from the end to trim off the incomplete character
 	toValidate := content
 	end := len(toValidate) - 1
 
-	if end < 0 {
-		// no-op
-	} else if toValidate[end]>>5 == 0b110 {
-		// Incomplete 1 byte extension e.g. © <c2><a9> which has been truncated to <c2>
-		toValidate = toValidate[:end]
-	} else if end > 0 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>4 == 0b1110 {
-		// Incomplete 2 byte extension e.g. ⛔ <e2><9b><94> which has been truncated to <e2><9b>
-		toValidate = toValidate[:end-1]
-	} else if end > 1 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>6 == 0b10 && toValidate[end-2]>>3 == 0b11110 {
-		// Incomplete 3 byte extension e.g. 💩 <f0><9f><92><a9> which has been truncated to <f0><9f><92>
-		toValidate = toValidate[:end-2]
+	// U+0000   U+007F 	  0yyyzzzz
+	// U+0080   U+07FF 	  110xxxyy 	10yyzzzz
+	// U+0800   U+FFFF 	  1110wwww 	10xxxxyy 	10yyzzzz
+	// U+010000 U+10FFFF 	11110uvv 	10vvwwww 	10xxxxyy 	10yyzzzz
+	cnt := 0
+	for end >= 0 && cnt < 4 {
+		c := toValidate[end]
+		if c>>5 == 0b110 || c>>4 == 0b1110 || c>>3 == 0b11110 {
+			// a leading byte
+			toValidate = toValidate[:end]
+			break
+		} else if c>>6 == 0b10 {
+			// a continuation byte
+			end--
+		} else {
+			// not an utf-8 byte
+			break
+		}
+		cnt++
 	}
+
 	if utf8.Valid(toValidate) {
-		log.Debug("Detected encoding: utf-8 (fast)")
 		return "UTF-8", nil
 	}
 
@@ -160,7 +148,7 @@ func DetectEncoding(content []byte) (string, error) {
 	if len(content) < 1024 {
 		// Check if original content is valid
 		if _, err := textDetector.DetectBest(content); err != nil {
-			return "", err
+			return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err
 		}
 		times := 1024 / len(content)
 		detectContent = make([]byte, 0, times*len(content))
@@ -171,14 +159,10 @@ func DetectEncoding(content []byte) (string, error) {
 		detectContent = content
 	}
 
-	// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break
+	// Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie-break
 	results, err := textDetector.DetectAll(detectContent)
 	if err != nil {
-		if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 {
-			log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
-			return setting.Repository.AnsiCharset, nil
-		}
-		return "", err
+		return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err
 	}
 
 	topConfidence := results[0].Confidence
@@ -201,11 +185,9 @@ func DetectEncoding(content []byte) (string, error) {
 	}
 
 	// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
-	if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
-		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
+	if topResult.Charset != "UTF-8" && setting.Repository.AnsiCharset != "" {
 		return setting.Repository.AnsiCharset, err
 	}
 
-	log.Debug("Detected encoding: %s", topResult.Charset)
-	return topResult.Charset, err
+	return topResult.Charset, nil
 }