Files
pages/internal/server/content_detect.go
Anya Tokugawa c1175d389f Implement Knowledge City platform, organizer and DAV integration
Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming.

Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
2026-09-27 14:44:43 +09:00

189 lines
5.2 KiB
Go

package server
import (
"bytes"
"encoding/json"
"mime"
"net/http"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"
"golang.org/x/text/encoding/japanese"
textunicode "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
type detectedText struct {
ContentType string
Text string
IsText bool
}
func detectRevisionText(data []byte, filename string) detectedText {
if len(data) == 0 {
return detectedText{"text/plain; charset=utf-8", "", true}
}
magicType := http.DetectContentType(data)
if magicType != "application/octet-stream" &&
!strings.HasPrefix(magicType, "text/plain") {
return detectedText{ContentType: magicType}
}
decoded, charset, ok := decodeTextBytes(data)
if !ok {
return detectedText{ContentType: http.DetectContentType(data)}
}
contentType := detectTextMediaType(decoded, filename)
if strings.HasPrefix(contentType, "text/") ||
contentType == "application/json" ||
contentType == "application/xml" ||
contentType == "application/javascript" ||
contentType == "application/x-sh" {
contentType += "; charset=" + charset
}
return detectedText{
ContentType: contentType,
Text: decoded,
IsText: true,
}
}
func decodeTextBytes(data []byte) (string, string, bool) {
if bytes.HasPrefix(data, []byte{0xef, 0xbb, 0xbf}) {
return string(data[3:]), "utf-8", true
}
if bytes.HasPrefix(data, []byte{0xff, 0xfe}) {
decoded, _, err := transform.Bytes(
textunicode.UTF16(textunicode.LittleEndian, textunicode.ExpectBOM).NewDecoder(),
data,
)
return string(decoded), "utf-16le", err == nil
}
if bytes.HasPrefix(data, []byte{0xfe, 0xff}) {
decoded, _, err := transform.Bytes(
textunicode.UTF16(textunicode.BigEndian, textunicode.ExpectBOM).NewDecoder(),
data,
)
return string(decoded), "utf-16be", err == nil
}
if utf8.Valid(data) {
text := string(data)
return text, "utf-8", looksLikeText(text)
}
if bytes.Contains(data, []byte{0x1b, '$'}) {
if decoded, _, err := transform.Bytes(japanese.ISO2022JP.NewDecoder(), data); err == nil {
text := string(decoded)
return text, "iso-2022-jp", looksLikeText(text)
}
}
for _, candidate := range []struct {
name string
enc transform.Transformer
}{
{"shift_jis", japanese.ShiftJIS.NewDecoder()},
{"euc-jp", japanese.EUCJP.NewDecoder()},
} {
decoded, _, err := transform.Bytes(candidate.enc, data)
if err == nil && looksLikeText(string(decoded)) {
return string(decoded), candidate.name, true
}
}
return "", "", false
}
func looksLikeText(text string) bool {
if text == "" {
return true
}
var controls, total int
for _, r := range text {
total++
if r == '\n' || r == '\r' || r == '\t' || r == '\f' {
continue
}
if unicode.IsControl(r) || r == utf8.RuneError {
controls++
}
}
return controls*100 <= total*2
}
func detectTextMediaType(text, filename string) string {
firstLine := text
if newline := strings.IndexByte(firstLine, '\n'); newline >= 0 {
firstLine = firstLine[:newline]
}
lower := strings.ToLower(firstLine)
switch {
case strings.HasPrefix(lower, "#!") &&
strings.Contains(lower, "python"):
return "text/x-python"
case strings.HasPrefix(lower, "#!") &&
(strings.Contains(lower, "/sh") ||
strings.Contains(lower, "bash") ||
strings.Contains(lower, "zsh")):
return "application/x-sh"
case strings.HasPrefix(lower, "#!") &&
(strings.Contains(lower, "node") || strings.Contains(lower, "deno")):
return "application/javascript"
}
if json.Valid([]byte(text)) {
return "application/json"
}
if strings.HasPrefix(strings.TrimSpace(text), "<?xml") {
return "application/xml"
}
if extType := mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))); extType != "" {
if mediaType, _, err := mime.ParseMediaType(extType); err == nil {
return mediaType
}
}
return "text/plain"
}
func codeLanguage(contentType, filename, text string) string {
mediaType, _, _ := mime.ParseMediaType(contentType)
ext := strings.ToLower(filepath.Ext(filename))
switch {
case mediaType == "text/x-python" || ext == ".py":
return "python"
case mediaType == "application/x-sh" || mediaType == "application/x-shellscript" ||
ext == ".sh" || ext == ".bash" || ext == ".zsh":
return "shell"
case mediaType == "application/json" || ext == ".json":
return "json"
case mediaType == "application/javascript" || mediaType == "application/x-javascript" ||
ext == ".js" || ext == ".jsx" || ext == ".mjs" || ext == ".cjs":
return "javascript"
case ext == ".ts" || ext == ".tsx":
return "typescript"
case mediaType == "text/html" || ext == ".html" || ext == ".htm":
return "html"
case ext == ".css":
return "css"
case ext == ".go":
return "go"
case ext == ".rs":
return "rust"
case ext == ".java":
return "java"
case ext == ".c" || ext == ".h" || ext == ".cc" || ext == ".cpp" || ext == ".hpp":
return "cpp"
case mediaType == "application/x-yaml" || mediaType == "application/yaml" ||
ext == ".yaml" || ext == ".yml":
return "yaml"
case mediaType == "application/toml" || ext == ".toml":
return "toml"
case ext == ".sql":
return "sql"
}
detected := detectTextMediaType(text, filename)
if detected != "text/plain" && detected != mediaType {
return codeLanguage(detected, filename, text)
}
return "text"
}