From 5a44be627c055d3e9eb406ec4a91579de78b6910 Mon Sep 17 00:00:00 2001
From: guillep2k <18600385+guillep2k@users.noreply.github.com>
Date: Thu, 15 Aug 2019 09:07:28 -0300
Subject: [PATCH] Convert files to utf-8 for indexing (#7814)
* Convert files to utf-8 for indexing
* Move utf8 functions to modules/base
* Bump repoIndexerLatestVersion to 3
* Add tests for base/encoding.go
* Changes to pass gosimple
* Move UTF8 funcs into new modules/charset package
---
integrations/migration-test/migration_test.go | 3 +-
models/git_diff.go | 8 +-
models/repo_indexer.go | 4 +-
modules/base/tool.go | 49 -----
modules/base/tool_test.go | 36 ----
modules/charset/charset.go | 152 ++++++++++++++
modules/charset/charset_test.go | 191 ++++++++++++++++++
modules/indexer/repo.go | 2 +-
modules/repofiles/update.go | 20 +-
modules/templates/helper.go | 56 -----
routers/repo/commit.go | 4 +-
routers/repo/editor.go | 4 +-
routers/repo/view.go | 8 +-
13 files changed, 371 insertions(+), 166 deletions(-)
create mode 100644 modules/charset/charset.go
create mode 100644 modules/charset/charset_test.go
diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go
index fd0d0b7c72..858f339b89 100644
--- a/integrations/migration-test/migration_test.go
+++ b/integrations/migration-test/migration_test.go
@@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/setting"
"github.com/go-xorm/xorm"
@@ -106,7 +107,7 @@ func readSQLFromFile(version string) (string, error) {
if err != nil {
return "", err
}
- return string(base.RemoveBOMIfPresent(bytes)), nil
+ return string(charset.RemoveBOMIfPresent(bytes)), nil
}
func restoreOldDB(t *testing.T, version string) bool {
diff --git a/models/git_diff.go b/models/git_diff.go
index 518d543e7b..62bb35be07 100644
--- a/models/git_diff.go
+++ b/models/git_diff.go
@@ -19,7 +19,7 @@ import (
"strconv"
"strings"
- "code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/highlight"
"code.gitea.io/gitea/modules/log"
@@ -27,7 +27,7 @@ import (
"code.gitea.io/gitea/modules/setting"
"github.com/Unknwon/com"
"github.com/sergi/go-diff/diffmatchpatch"
- "golang.org/x/net/html/charset"
+ stdcharset "golang.org/x/net/html/charset"
"golang.org/x/text/transform"
)
@@ -641,9 +641,9 @@ func ParsePatch(maxLines, maxLineCharacters, maxFiles int, reader io.Reader) (*D
buf.WriteString("\n")
}
}
- charsetLabel, err := base.DetectEncoding(buf.Bytes())
+ charsetLabel, err := charset.DetectEncoding(buf.Bytes())
if charsetLabel != "UTF-8" && err == nil {
- encoding, _ := charset.Lookup(charsetLabel)
+ encoding, _ := stdcharset.Lookup(charsetLabel)
if encoding != nil {
d := encoding.NewDecoder()
for _, sec := range f.Sections {
diff --git a/models/repo_indexer.go b/models/repo_indexer.go
index c991b1aac9..f625b80389 100644
--- a/models/repo_indexer.go
+++ b/models/repo_indexer.go
@@ -10,6 +10,7 @@ import (
"strings"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/log"
@@ -207,6 +208,7 @@ func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch)
if err != nil {
return err
} else if !base.IsTextFile(fileContents) {
+ // FIXME: UTF-16 files will probably fail here
return nil
}
indexerUpdate := indexer.RepoIndexerUpdate{
@@ -214,7 +216,7 @@ func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch)
Op: indexer.RepoIndexerOpUpdate,
Data: &indexer.RepoIndexerData{
RepoID: repo.ID,
- Content: string(fileContents),
+ Content: string(charset.ToUTF8DropErrors(fileContents)),
},
}
return indexerUpdate.AddToFlushingBatch(batch)
diff --git a/modules/base/tool.go b/modules/base/tool.go
index 4893abff71..2bb39dbfea 100644
--- a/modules/base/tool.go
+++ b/modules/base/tool.go
@@ -5,7 +5,6 @@
package base
import (
- "bytes"
"crypto/md5"
"crypto/rand"
"crypto/sha1"
@@ -26,7 +25,6 @@ import (
"strings"
"time"
"unicode"
- "unicode/utf8"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
@@ -35,12 +33,8 @@ import (
"github.com/Unknwon/com"
"github.com/Unknwon/i18n"
- "github.com/gogits/chardet"
)
-// UTF8BOM is the utf-8 byte-order marker
-var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
-
// EncodeMD5 encodes string to md5 hex value.
func EncodeMD5(str string) string {
m := md5.New()
@@ -68,49 +62,6 @@ func ShortSha(sha1 string) string {
return TruncateString(sha1, 10)
}
-// DetectEncoding detect the encoding of content
-func DetectEncoding(content []byte) (string, error) {
- if utf8.Valid(content) {
- log.Debug("Detected encoding: utf-8 (fast)")
- return "UTF-8", nil
- }
-
- textDetector := chardet.NewTextDetector()
- var detectContent []byte
- if len(content) < 1024 {
- // Check if original content is valid
- if _, err := textDetector.DetectBest(content); err != nil {
- return "", err
- }
- times := 1024 / len(content)
- detectContent = make([]byte, 0, times*len(content))
- for i := 0; i < times; i++ {
- detectContent = append(detectContent, content...)
- }
- } else {
- detectContent = content
- }
- result, err := textDetector.DetectBest(detectContent)
- if err != nil {
- return "", err
- }
- if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
- log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
- return setting.Repository.AnsiCharset, err
- }
-
- log.Debug("Detected encoding: %s", result.Charset)
- return result.Charset, err
-}
-
-// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
-func RemoveBOMIfPresent(content []byte) []byte {
- if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
- return content[3:]
- }
- return content
-}
-
// BasicAuthDecode decode basic auth string
func BasicAuthDecode(encoded string) (string, string, error) {
s, err := base64.StdEncoding.DecodeString(encoded)
diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go
index fa61e5dfb1..3f1eebfc4b 100644
--- a/modules/base/tool_test.go
+++ b/modules/base/tool_test.go
@@ -64,42 +64,6 @@ func TestShortSha(t *testing.T) {
assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
}
-func TestDetectEncoding(t *testing.T) {
- testSuccess := func(b []byte, expected string) {
- encoding, err := DetectEncoding(b)
- assert.NoError(t, err)
- assert.Equal(t, expected, encoding)
- }
- // utf-8
- b := []byte("just some ascii")
- testSuccess(b, "UTF-8")
-
- // utf-8-sig: "hey" (with BOM)
- b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
- testSuccess(b, "UTF-8")
-
- // utf-16: "hey<accented G>"
- b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
- testSuccess(b, "UTF-16LE")
-
- // iso-8859-1: d<accented e>cor<newline>
- b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
- encoding, err := DetectEncoding(b)
- assert.NoError(t, err)
- // due to a race condition in `chardet` library, it could either detect
- // "ISO-8859-1" or "IS0-8859-2" here. Technically either is correct, so
- // we accept either.
- assert.Contains(t, encoding, "ISO-8859")
-
- setting.Repository.AnsiCharset = "placeholder"
- testSuccess(b, "placeholder")
-
- // invalid bytes
- b = []byte{0xfa}
- _, err = DetectEncoding(b)
- assert.Error(t, err)
-}
-
func TestBasicAuthDecode(t *testing.T) {
_, _, err := BasicAuthDecode("?")
assert.Equal(t, "illegal base64 data at input byte 0", err.Error())
diff --git a/modules/charset/charset.go b/modules/charset/charset.go
new file mode 100644
index 0000000000..f43fe387ed
--- /dev/null
+++ b/modules/charset/charset.go
@@ -0,0 +1,152 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+ "bytes"
+ "fmt"
+ "unicode/utf8"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/gogits/chardet"
+ "golang.org/x/net/html/charset"
+ "golang.org/x/text/transform"
+)
+
+// UTF8BOM is the utf-8 byte-order marker
+var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
+
+// ToUTF8WithErr converts content to UTF8 encoding
+func ToUTF8WithErr(content []byte) (string, error) {
+ charsetLabel, err := DetectEncoding(content)
+ if err != nil {
+ return "", err
+ } else if charsetLabel == "UTF-8" {
+ return string(RemoveBOMIfPresent(content)), nil
+ }
+
+ encoding, _ := charset.Lookup(charsetLabel)
+ if encoding == nil {
+ return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
+ }
+
+ // If there is an error, we concatenate the nicely decoded part and the
+ // original left over. This way we won't lose data.
+ result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+ if err != nil {
+ result = append(result, content[n:]...)
+ }
+
+ result = RemoveBOMIfPresent(result)
+
+ return string(result), err
+}
+
+// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
+func ToUTF8WithFallback(content []byte) []byte {
+ charsetLabel, err := DetectEncoding(content)
+ if err != nil || charsetLabel == "UTF-8" {
+ return RemoveBOMIfPresent(content)
+ }
+
+ encoding, _ := charset.Lookup(charsetLabel)
+ if encoding == nil {
+ return content
+ }
+
+ // If there is an error, we concatenate the nicely decoded part and the
+ // original left over. This way we won't lose data.
+ result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+ if err != nil {
+ return append(result, content[n:]...)
+ }
+
+ return RemoveBOMIfPresent(result)
+}
+
+// ToUTF8 converts content to UTF8 encoding and ignore error
+func ToUTF8(content string) string {
+ res, _ := ToUTF8WithErr([]byte(content))
+ return res
+}
+
+// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
+func ToUTF8DropErrors(content []byte) []byte {
+ charsetLabel, err := DetectEncoding(content)
+ if err != nil || charsetLabel == "UTF-8" {
+ return RemoveBOMIfPresent(content)
+ }
+
+ encoding, _ := charset.Lookup(charsetLabel)
+ if encoding == nil {
+ return content
+ }
+
+ // We ignore any non-decodable parts from the file.
+ // Some parts might be lost
+ var decoded []byte
+ decoder := encoding.NewDecoder()
+ idx := 0
+ for {
+ result, n, err := transform.Bytes(decoder, content[idx:])
+ decoded = append(decoded, result...)
+ if err == nil {
+ break
+ }
+ decoded = append(decoded, ' ')
+ idx = idx + n + 1
+ if idx >= len(content) {
+ break
+ }
+ }
+
+ return RemoveBOMIfPresent(decoded)
+}
+
+// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
+func RemoveBOMIfPresent(content []byte) []byte {
+ if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
+ return content[3:]
+ }
+ return content
+}
+
+// DetectEncoding detect the encoding of content
+func DetectEncoding(content []byte) (string, error) {
+ if utf8.Valid(content) {
+ log.Debug("Detected encoding: utf-8 (fast)")
+ return "UTF-8", nil
+ }
+
+ textDetector := chardet.NewTextDetector()
+ var detectContent []byte
+ if len(content) < 1024 {
+ // Check if original content is valid
+ if _, err := textDetector.DetectBest(content); err != nil {
+ return "", err
+ }
+ times := 1024 / len(content)
+ detectContent = make([]byte, 0, times*len(content))
+ for i := 0; i < times; i++ {
+ detectContent = append(detectContent, content...)
+ }
+ } else {
+ detectContent = content
+ }
+ result, err := textDetector.DetectBest(detectContent)
+ if err != nil {
+ return "", err
+ }
+ // FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
+ if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
+ log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
+ return setting.Repository.AnsiCharset, err
+ }
+
+ log.Debug("Detected encoding: %s", result.Charset)
+ return result.Charset, err
+}
diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go
new file mode 100644
index 0000000000..fc9ec6a979
--- /dev/null
+++ b/modules/charset/charset_test.go
@@ -0,0 +1,191 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+ "testing"
+
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestRemoveBOMIfPresent(t *testing.T) {
+ res := RemoveBOMIfPresent([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+ res = RemoveBOMIfPresent([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+}
+
+func TestToUTF8WithErr(t *testing.T) {
+ var res string
+ var err error
+
+ res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
+ assert.Equal(t, "ABC", res)
+ assert.NoError(t, err)
+
+ res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, "áéíóú", res)
+ assert.NoError(t, err)
+
+ res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, "áéíóú", res)
+ assert.NoError(t, err)
+
+ // This test FAILS
+ res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+ assert.Equal(t, "Hola, así cómo ños", res)
+ assert.NoError(t, err)
+
+ res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Regexp(t, "^Hola, así cómo", res)
+ assert.NoError(t, err)
+
+ res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Regexp(t, "^Hola, así cómo", res)
+ assert.NoError(t, err)
+
+ // Japanese (Shift-JIS)
+ res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+ assert.Equal(t, "日属秘ぞしちゅ。", res)
+ assert.NoError(t, err)
+
+ res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
+ assert.Equal(t, "\x00\x00\x00\x00", res)
+ assert.NoError(t, err)
+}
+
+func TestToUTF8WithFallback(t *testing.T) {
+ res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43})
+ assert.Equal(t, []byte("ABC"), res)
+
+ res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, []byte("áéíóú"), res)
+
+ res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, []byte("áéíóú"), res)
+
+ res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+ assert.Equal(t, []byte("Hola, así cómo ños"), res)
+
+ minmatch := []byte("Hola, así cómo ")
+
+ res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+ res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+ // Japanese (Shift-JIS)
+ res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+ assert.Equal(t, []byte("日属秘ぞしちゅ。"), res)
+
+ res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00})
+ assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestToUTF8(t *testing.T) {
+ res := ToUTF8("ABC")
+ assert.Equal(t, "ABC", res)
+
+ res = ToUTF8("áéíóú")
+ assert.Equal(t, "áéíóú", res)
+
+ // With utf-8 BOM
+ res = ToUTF8("\ufeffáéíóú")
+ assert.Equal(t, "áéíóú", res)
+
+ res = ToUTF8("Hola, así cómo ños")
+ assert.Equal(t, "Hola, así cómo ños", res)
+
+ res = ToUTF8("Hola, así cómo \x07ños")
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Regexp(t, "^Hola, así cómo", res)
+
+ // This test FAILS
+ // res = ToUTF8("Hola, así cómo \x81ños")
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ // assert.Regexp(t, "^Hola, así cómo", res)
+
+ // Japanese (Shift-JIS)
+ res = ToUTF8("\x93\xFA\x91\xAE\x94\xE9\x82\xBC\x82\xB5\x82\xBF\x82\xE3\x81\x42")
+ assert.Equal(t, "日属秘ぞしちゅ。", res)
+
+ res = ToUTF8("\x00\x00\x00\x00")
+ assert.Equal(t, "\x00\x00\x00\x00", res)
+}
+
+func TestToUTF8DropErrors(t *testing.T) {
+ res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
+ assert.Equal(t, []byte("ABC"), res)
+
+ res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, []byte("áéíóú"), res)
+
+ res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+ assert.Equal(t, []byte("áéíóú"), res)
+
+ res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+ assert.Equal(t, []byte("Hola, así cómo ños"), res)
+
+ minmatch := []byte("Hola, así cómo ")
+
+ res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+ res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+ // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+ assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+ // Japanese (Shift-JIS)
+ res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+ assert.Equal(t, []byte("日属秘ぞしちゅ。"), res)
+
+ res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
+ assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestDetectEncoding(t *testing.T) {
+ testSuccess := func(b []byte, expected string) {
+ encoding, err := DetectEncoding(b)
+ assert.NoError(t, err)
+ assert.Equal(t, expected, encoding)
+ }
+ // utf-8
+ b := []byte("just some ascii")
+ testSuccess(b, "UTF-8")
+
+ // utf-8-sig: "hey" (with BOM)
+ b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
+ testSuccess(b, "UTF-8")
+
+ // utf-16: "hey<accented G>"
+ b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
+ testSuccess(b, "UTF-16LE")
+
+ // iso-8859-1: d<accented e>cor<newline>
+ b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
+ encoding, err := DetectEncoding(b)
+ assert.NoError(t, err)
+ // due to a race condition in `chardet` library, it could either detect
+ // "ISO-8859-1" or "IS0-8859-2" here. Technically either is correct, so
+ // we accept either.
+ assert.Contains(t, encoding, "ISO-8859")
+
+ setting.Repository.AnsiCharset = "placeholder"
+ testSuccess(b, "placeholder")
+
+ // invalid bytes
+ b = []byte{0xfa}
+ _, err = DetectEncoding(b)
+ assert.Error(t, err)
+}
diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go
index 787f9a9467..645a6fc88b 100644
--- a/modules/indexer/repo.go
+++ b/modules/indexer/repo.go
@@ -23,7 +23,7 @@ const (
repoIndexerAnalyzer = "repoIndexerAnalyzer"
repoIndexerDocType = "repoIndexerDocType"
- repoIndexerLatestVersion = 2
+ repoIndexerLatestVersion = 3
)
// repoIndexer (thread-safe) index for repository contents
diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go
index 62e2afbc96..ee1b16bce9 100644
--- a/modules/repofiles/update.go
+++ b/modules/repofiles/update.go
@@ -11,17 +11,17 @@ import (
"path"
"strings"
- "golang.org/x/net/html/charset"
- "golang.org/x/text/transform"
-
"code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/cache"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
+
+ stdcharset "golang.org/x/net/html/charset"
+ "golang.org/x/text/transform"
)
// IdentityOptions for a person's identity like an author or committer
@@ -87,15 +87,15 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
}
- encoding, err := base.DetectEncoding(buf)
+ encoding, err := charset.DetectEncoding(buf)
if err != nil {
// just default to utf-8 and no bom
return "UTF-8", false
}
if encoding == "UTF-8" {
- return encoding, bytes.Equal(buf[0:3], base.UTF8BOM)
+ return encoding, bytes.Equal(buf[0:3], charset.UTF8BOM)
}
- charsetEncoding, _ := charset.Lookup(encoding)
+ charsetEncoding, _ := stdcharset.Lookup(encoding)
if charsetEncoding == nil {
return "UTF-8", false
}
@@ -107,7 +107,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
}
if n > 2 {
- return encoding, bytes.Equal([]byte(result)[0:3], base.UTF8BOM)
+ return encoding, bytes.Equal([]byte(result)[0:3], charset.UTF8BOM)
}
return encoding, false
@@ -321,10 +321,10 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
content := opts.Content
if bom {
- content = string(base.UTF8BOM) + content
+ content = string(charset.UTF8BOM) + content
}
if encoding != "UTF-8" {
- charsetEncoding, _ := charset.Lookup(encoding)
+ charsetEncoding, _ := stdcharset.Lookup(encoding)
if charsetEncoding != nil {
result, _, err := transform.String(charsetEncoding.NewEncoder(), content)
if err != nil {
diff --git a/modules/templates/helper.go b/modules/templates/helper.go
index ceb4501c5b..17df375a80 100644
--- a/modules/templates/helper.go
+++ b/modules/templates/helper.go
@@ -28,8 +28,6 @@ import (
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/setting"
- "golang.org/x/net/html/charset"
- "golang.org/x/text/transform"
"gopkg.in/editorconfig/editorconfig-core-go.v1"
)
@@ -274,60 +272,6 @@ func Sha1(str string) string {
return base.EncodeSha1(str)
}
-// ToUTF8WithErr converts content to UTF8 encoding
-func ToUTF8WithErr(content []byte) (string, error) {
- charsetLabel, err := base.DetectEncoding(content)
- if err != nil {
- return "", err
- } else if charsetLabel == "UTF-8" {
- return string(base.RemoveBOMIfPresent(content)), nil
- }
-
- encoding, _ := charset.Lookup(charsetLabel)
- if encoding == nil {
- return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
- }
-
- // If there is an error, we concatenate the nicely decoded part and the
- // original left over. This way we won't lose data.
- result, n, err := transform.Bytes(encoding.NewDecoder(), content)
- if err != nil {
- result = append(result, content[n:]...)
- }
-
- result = base.RemoveBOMIfPresent(result)
-
- return string(result), err
-}
-
-// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
-func ToUTF8WithFallback(content []byte) []byte {
- charsetLabel, err := base.DetectEncoding(content)
- if err != nil || charsetLabel == "UTF-8" {
- return base.RemoveBOMIfPresent(content)
- }
-
- encoding, _ := charset.Lookup(charsetLabel)
- if encoding == nil {
- return content
- }
-
- // If there is an error, we concatenate the nicely decoded part and the
- // original left over. This way we won't lose data.
- result, n, err := transform.Bytes(encoding.NewDecoder(), content)
- if err != nil {
- return append(result, content[n:]...)
- }
-
- return base.RemoveBOMIfPresent(result)
-}
-
-// ToUTF8 converts content to UTF8 encoding and ignore error
-func ToUTF8(content string) string {
- res, _ := ToUTF8WithErr([]byte(content))
- return res
-}
-
// ReplaceLeft replaces all prefixes 'oldS' in 's' with 'newS'.
func ReplaceLeft(s, oldS, newS string) string {
oldLen, newLen, i, n := len(oldS), len(newS), 0, 0
diff --git a/routers/repo/commit.go b/routers/repo/commit.go
index 4dbedea2a0..79fa024dc9 100644
--- a/routers/repo/commit.go
+++ b/routers/repo/commit.go
@@ -11,11 +11,11 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/templates"
)
const (
@@ -251,7 +251,7 @@ func Diff(ctx *context.Context) {
note := &git.Note{}
err = git.GetNote(ctx.Repo.GitRepo, commitID, note)
if err == nil {
- ctx.Data["Note"] = string(templates.ToUTF8WithFallback(note.Message))
+ ctx.Data["Note"] = string(charset.ToUTF8WithFallback(note.Message))
ctx.Data["NoteCommit"] = note.Commit
ctx.Data["NoteAuthor"] = models.ValidateCommitWithEmail(note.Commit)
}
diff --git a/routers/repo/editor.go b/routers/repo/editor.go
index 1c7eee98ac..68b94662dc 100644
--- a/routers/repo/editor.go
+++ b/routers/repo/editor.go
@@ -14,12 +14,12 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/repofiles"
"code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/templates"
"code.gitea.io/gitea/modules/upload"
"code.gitea.io/gitea/modules/util"
)
@@ -118,7 +118,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
d, _ := ioutil.ReadAll(dataRc)
buf = append(buf, d...)
- if content, err := templates.ToUTF8WithErr(buf); err != nil {
+ if content, err := charset.ToUTF8WithErr(buf); err != nil {
log.Error("ToUTF8WithErr: %v", err)
ctx.Data["FileContent"] = string(buf)
} else {
diff --git a/routers/repo/view.go b/routers/repo/view.go
index edaf24017c..b534ae0e16 100644
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/highlight"
@@ -23,7 +24,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/templates"
)
const (
@@ -160,7 +160,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
ctx.Data["FileSize"] = fileSize
} else {
d, _ := ioutil.ReadAll(dataRc)
- buf = templates.ToUTF8WithFallback(append(buf, d...))
+ buf = charset.ToUTF8WithFallback(append(buf, d...))
if markup.Type(readmeFile.Name()) != "" {
ctx.Data["IsMarkup"] = true
@@ -278,7 +278,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
d, _ := ioutil.ReadAll(dataRc)
- buf = templates.ToUTF8WithFallback(append(buf, d...))
+ buf = charset.ToUTF8WithFallback(append(buf, d...))
readmeExist := markup.IsReadmeFile(blob.Name())
ctx.Data["ReadmeExist"] = readmeExist
@@ -293,7 +293,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
} else {
// Building code view blocks with line number on server side.
var fileContent string
- if content, err := templates.ToUTF8WithErr(buf); err != nil {
+ if content, err := charset.ToUTF8WithErr(buf); err != nil {
log.Error("ToUTF8WithErr: %v", err)
fileContent = string(buf)
} else {