package server import ( "bytes" "encoding/json" "mime" "net/http" "path/filepath" "strings" "unicode" "unicode/utf8" "golang.org/x/text/encoding/japanese" textunicode "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" ) type detectedText struct { ContentType string Text string IsText bool } func detectRevisionText(data []byte, filename string) detectedText { if len(data) == 0 { return detectedText{"text/plain; charset=utf-8", "", true} } magicType := http.DetectContentType(data) if magicType != "application/octet-stream" && !strings.HasPrefix(magicType, "text/plain") { return detectedText{ContentType: magicType} } decoded, charset, ok := decodeTextBytes(data) if !ok { return detectedText{ContentType: http.DetectContentType(data)} } contentType := detectTextMediaType(decoded, filename) if strings.HasPrefix(contentType, "text/") || contentType == "application/json" || contentType == "application/xml" || contentType == "application/javascript" || contentType == "application/x-sh" { contentType += "; charset=" + charset } return detectedText{ ContentType: contentType, Text: decoded, IsText: true, } } func decodeTextBytes(data []byte) (string, string, bool) { if bytes.HasPrefix(data, []byte{0xef, 0xbb, 0xbf}) { return string(data[3:]), "utf-8", true } if bytes.HasPrefix(data, []byte{0xff, 0xfe}) { decoded, _, err := transform.Bytes( textunicode.UTF16(textunicode.LittleEndian, textunicode.ExpectBOM).NewDecoder(), data, ) return string(decoded), "utf-16le", err == nil } if bytes.HasPrefix(data, []byte{0xfe, 0xff}) { decoded, _, err := transform.Bytes( textunicode.UTF16(textunicode.BigEndian, textunicode.ExpectBOM).NewDecoder(), data, ) return string(decoded), "utf-16be", err == nil } if utf8.Valid(data) { text := string(data) return text, "utf-8", looksLikeText(text) } if bytes.Contains(data, []byte{0x1b, '$'}) { if decoded, _, err := transform.Bytes(japanese.ISO2022JP.NewDecoder(), data); err == nil { text := string(decoded) return text, "iso-2022-jp", looksLikeText(text) } } for _, candidate := range []struct { name string enc transform.Transformer }{ {"shift_jis", japanese.ShiftJIS.NewDecoder()}, {"euc-jp", japanese.EUCJP.NewDecoder()}, } { decoded, _, err := transform.Bytes(candidate.enc, data) if err == nil && looksLikeText(string(decoded)) { return string(decoded), candidate.name, true } } return "", "", false } func looksLikeText(text string) bool { if text == "" { return true } var controls, total int for _, r := range text { total++ if r == '\n' || r == '\r' || r == '\t' || r == '\f' { continue } if unicode.IsControl(r) || r == utf8.RuneError { controls++ } } return controls*100 <= total*2 } func detectTextMediaType(text, filename string) string { firstLine := text if newline := strings.IndexByte(firstLine, '\n'); newline >= 0 { firstLine = firstLine[:newline] } lower := strings.ToLower(firstLine) switch { case strings.HasPrefix(lower, "#!") && strings.Contains(lower, "python"): return "text/x-python" case strings.HasPrefix(lower, "#!") && (strings.Contains(lower, "/sh") || strings.Contains(lower, "bash") || strings.Contains(lower, "zsh")): return "application/x-sh" case strings.HasPrefix(lower, "#!") && (strings.Contains(lower, "node") || strings.Contains(lower, "deno")): return "application/javascript" } if json.Valid([]byte(text)) { return "application/json" } if strings.HasPrefix(strings.TrimSpace(text), "