Files
pages/internal/server/import_handler.go
T
Anya Tokugawa c1175d389f Implement Knowledge City platform, organizer and DAV integration
Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming.

Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
2026-09-27 14:44:43 +09:00

829 lines
24 KiB
Go

package server
import (
"archive/tar"
"archive/zip"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"pageswiki/internal/render"
"pageswiki/internal/store"
)
const importAnalyzeEventType = "import.analyze"
const importCommitEventType = "import.commit"
const maxImportArchiveBytes = int64(512 << 20)
const maxImportFiles = 10000
const maxImportExpandedBytes = int64(2 << 30)
type importAnalyzePayload struct{ DomainUUID, SessionUUID string }
type importCommitPayload struct{ DomainUUID, SessionUUID, Conflict string }
func (s *Server) initializeImportRoot() {
base := filepath.Dir(s.store.DocumentRoot)
if s.store.DocumentRoot == "" {
base = "data"
}
s.importRoot = filepath.Join(base, "imports")
if err := os.MkdirAll(s.importRoot, 0700); err != nil {
s.importRoot = ""
}
}
func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
domainUUID := s.domainUUIDForRequest(r)
if r.Method == http.MethodPost {
s.handleImportUpload(w, r, domainUUID)
return
}
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/v/import"), "/")
data := map[string]any{"Title": translate(s.getLocale(domainUUID), "import.title"), "Locale": s.getLocale(domainUUID)}
if id == "" {
sessions, err := s.store.ListImportSessions(domainUUID)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
data["Sessions"] = sessions
} else {
session, err := s.store.ImportSession(domainUUID, id)
if err != nil {
http.NotFound(w, r)
return
}
items, err := s.store.ListImportItems(id)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
s.mergeExistingCitySuggestions(domainUUID, session.TargetPrefix, items)
if session.Status == "failed" {
pending := items[:0]
for _, item := range items {
if !item.Imported {
pending = append(pending, item)
}
}
items = pending
}
data["Session"], data["Items"] = session, items
}
s.renderWithContext(w, r, "import.gohtml", data)
}
func (s *Server) handleImportAPI(w http.ResponseWriter, r *http.Request) {
domainUUID := s.domainUUIDForRequest(r)
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/_/api/v1/import/"), "/")
if id == "" {
http.NotFound(w, r)
return
}
session, err := s.store.ImportSession(domainUUID, id)
if err != nil {
http.NotFound(w, r)
return
}
if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" {
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
if authErr != nil || (session.ActorUUID != principal.UUID && !s.store.IsBuiltinGroupAdmin(domainUUID, principal.UUID, "domain_admin")) {
writeEventJSON(w, http.StatusForbidden, map[string]string{"error": "import owner or domain admin required"})
return
}
}
if r.Method == http.MethodPatch {
var request struct {
Items []store.ImportItem `json:"items"`
}
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&request) != nil {
writeEventJSON(w, 400, map[string]string{"error": "invalid JSON"})
return
}
if err := s.store.UpdateImportPlan(domainUUID, id, request.Items); err != nil {
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
return
}
writeEventJSON(w, 200, map[string]string{"status": "ok"})
return
}
if r.Method == http.MethodPost {
var request struct {
Conflict string `json:"conflict"`
}
_ = json.NewDecoder(r.Body).Decode(&request)
if request.Conflict != "rename" {
request.Conflict = "skip"
}
if err := s.store.QueueImportCommit(domainUUID, id); err != nil {
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
return
}
payload, _ := json.Marshal(importCommitPayload{DomainUUID: domainUUID, SessionUUID: id, Conflict: request.Conflict})
queued, err := s.store.EnqueueEvent(importCommitEventType, id, string(payload), time.Now())
if err != nil || !queued {
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
writeEventJSON(w, 500, map[string]string{"error": "could not queue import"})
return
}
s.wakeEventScheduler()
writeEventJSON(w, 202, map[string]string{"status": "queued"})
return
}
if r.Method == http.MethodDelete {
if err := s.rollbackImport(domainUUID, id); err != nil {
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
return
}
writeEventJSON(w, 200, map[string]string{"status": "rolled_back"})
return
}
w.WriteHeader(405)
}
func (s *Server) handleImportUpload(w http.ResponseWriter, r *http.Request, domainUUID string) {
if s.importRoot == "" {
http.Error(w, "import storage unavailable", 500)
return
}
r.Body = http.MaxBytesReader(w, r.Body, maxImportArchiveBytes+(1<<20))
if err := r.ParseMultipartForm(32 << 20); err != nil {
http.Error(w, err.Error(), 413)
return
}
file, header, err := r.FormFile("archive")
if err != nil {
http.Error(w, "archive is required", 400)
return
}
defer file.Close()
name := filepath.Base(header.Filename)
if !supportedImportArchive(name) {
http.Error(w, "supported archives: .zip, .tar, .tar.gz, .tgz, .tar.xz, .txz", 400)
return
}
target := normalizeImportTarget(r.FormValue("target_prefix"))
actor := ""
mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode")
if mode == "enforcing" {
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
if authErr != nil || s.store.Authorize(store.AuthorizationRequest{DomainUUID: domainUUID, PrincipalUUID: principal.UUID, Capability: "upload", TargetType: "path", TargetID: target}) != nil {
http.Error(w, "forbidden", 403)
return
}
actor = principal.UUID
}
temp, err := os.CreateTemp(s.importRoot, "upload-*")
if err != nil {
http.Error(w, err.Error(), 500)
return
}
tempPath := temp.Name()
defer func() { _ = temp.Close() }()
n, err := io.Copy(temp, io.LimitReader(file, maxImportArchiveBytes+1))
if err != nil || n > maxImportArchiveBytes {
_ = os.Remove(tempPath)
http.Error(w, "archive is too large or unreadable", 413)
return
}
_ = temp.Close()
id, err := s.store.CreateImportSession(domainUUID, actor, name, tempPath, target)
if err != nil {
_ = os.Remove(tempPath)
http.Error(w, err.Error(), 500)
return
}
payload, _ := json.Marshal(importAnalyzePayload{DomainUUID: domainUUID, SessionUUID: id})
queued, err := s.store.EnqueueEvent(importAnalyzeEventType, id, string(payload), time.Now())
if err != nil || !queued {
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
http.Error(w, "could not queue import", 500)
return
}
s.wakeEventScheduler()
http.Redirect(w, r, "/v/import/"+id, 303)
}
func supportedImportArchive(name string) bool {
name = strings.ToLower(name)
for _, suffix := range []string{".zip", ".tar", ".tar.gz", ".tgz", ".tar.xz", ".txz"} {
if strings.HasSuffix(name, suffix) {
return true
}
}
return false
}
func normalizeImportTarget(value string) string {
value = "/" + strings.Trim(strings.TrimSpace(value), "/")
if value == "/" {
return value
}
return path.Clean(value)
}
func safeArchiveName(name string) (string, error) {
name = strings.ReplaceAll(name, "\\", "/")
if strings.HasPrefix(name, "/") || strings.ContainsRune(name, '\x00') {
return "", errors.New("unsafe archive path")
}
for _, part := range strings.Split(name, "/") {
if part == ".." {
return "", errors.New("unsafe archive path")
}
}
clean := path.Clean(name)
if clean == "" || clean == "." || strings.HasPrefix(clean, "../") {
return "", errors.New("unsafe archive path")
}
return clean, nil
}
func (s *Server) analyzeImport(payload importAnalyzePayload) error {
s.store.MarkImportAnalyzing(payload.SessionUUID)
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
if err != nil {
return err
}
items, err := scanImportArchive(session.ArchivePath, session.ArchiveName, session.TargetPrefix)
if err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
s.mergeExistingCitySuggestions(session.DomainUUID, session.TargetPrefix, items)
if err := s.store.ReplaceImportItems(session.UUID, items); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
return nil
}
func (s *Server) mergeExistingCitySuggestions(domainUUID, targetPrefix string, items []store.ImportItem) {
for i := range items {
source := strings.Split(strings.Trim(items[i].SourcePath, "/"), "/")
if len(source) < 2 {
continue
}
qPath := path.Join(targetPrefix, source[0])
q, err := s.store.CityNodeOverlappingPath(domainUUID, "quartier", "", qPath)
if err != nil {
q, err = s.store.CityNode(domainUUID, "quartier", items[i].Quartier)
}
if err != nil {
continue
}
items[i].Quartier = q.Name
streetPath := path.Join(qPath, source[1])
street, err := s.store.CityNodeOverlappingPath(domainUUID, "street", q.UUID, streetPath)
if err != nil {
continue
}
items[i].Street = street.Name
spotName := strings.TrimSpace(items[i].Spot)
spotPath := path.Join(targetPrefix, path.Dir(items[i].SourcePath))
if spotName == "" || spotName == street.Name {
spotPath = path.Join(streetPath, "_undefined")
} else if len(source) <= 3 {
spotPath = path.Join(streetPath, "_"+spotName)
}
spot, err := s.store.CityNodeOverlappingPath(domainUUID, "spot", street.UUID, spotPath)
if err == nil && spot.Name != street.Name+"-undefined" {
items[i].Spot = spot.Name
}
}
}
func scanImportArchive(filename, archiveName, target string) ([]store.ImportItem, error) {
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
return scanImportZIP(filename, target)
}
return scanImportTar(filename, archiveName, target)
}
func appendImportItem(items *[]store.ImportItem, name string, size int64, target string) error {
clean, err := safeArchiveName(name)
if err != nil {
return err
}
if strings.HasSuffix(clean, "/") {
return nil
}
if len(*items) >= maxImportFiles {
return errors.New("archive contains too many files")
}
parts := strings.Split(clean, "/")
title := ""
media := mime.TypeByExtension(strings.ToLower(filepath.Ext(clean)))
if strings.EqualFold(filepath.Ext(clean), ".md") {
media = "text/markdown"
title = strings.TrimSuffix(parts[len(parts)-1], filepath.Ext(clean))
}
if media == "" {
media = "application/octet-stream"
}
labels := parts
if len(labels) > 0 {
labels = labels[:len(labels)-1]
}
q, street, spot := "", "", ""
if len(labels) > 0 {
q = citySuggestionName(labels[0])
}
if len(labels) > 1 {
street = citySuggestionName(labels[1])
}
if len(labels) > 2 {
spot = citySuggestionName(labels[len(labels)-1])
}
confidence := 0.45
if media == "text/markdown" {
confidence = .7
}
targetPath := path.Join(target, clean)
if media == "text/markdown" {
targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath))
}
*items = append(*items, store.ImportItem{Sequence: len(*items) + 1, SourcePath: clean, TargetPath: targetPath, Size: size, ContentType: media, Title: title, Quartier: q, Street: street, Spot: spot, Confidence: confidence, Selected: true})
return nil
}
func (s *Server) commitImport(payload importCommitPayload) error {
s.store.MarkImportStatus(payload.SessionUUID, "importing")
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
if err != nil {
return err
}
items, err := s.store.ListImportItems(payload.SessionUUID)
if err != nil {
return err
}
wanted := map[string]bool{}
for _, x := range items {
if x.Selected && !x.Imported {
wanted[x.SourcePath] = true
}
}
journalEntries, err := s.store.ImportJournal(payload.SessionUUID)
if err != nil {
return err
}
journal := map[int]store.ImportJournalEntry{}
for _, entry := range journalEntries {
journal[entry.Sequence] = entry
}
contents, err := readImportArchive(session.ArchivePath, session.ArchiveName, wanted)
if err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
cityState := &importCityState{quartiers: map[string]string{}, streets: map[string]string{}, spots: map[string]string{}}
for _, item := range items {
if !item.Selected || item.Imported {
continue
}
body, ok := contents[item.SourcePath]
if !ok {
err = fmt.Errorf("archive entry disappeared: %s", item.SourcePath)
s.store.FailImportSession(session.UUID, err)
return err
}
target := item.TargetPath
if item.ContentType == "text/markdown" {
target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown")
}
prior, hasPrior := journal[item.Sequence]
if hasPrior {
target = prior.Path
}
if !hasPrior {
if _, resolveErr := s.store.ResolvePathInDomain(payload.DomainUUID, target); resolveErr == nil {
if payload.Conflict == "skip" {
continue
}
target = nextImportPath(s.store, payload.DomainUUID, target)
}
}
objectUUID := ""
var createErr error
if hasPrior {
objectUUID = prior.ObjectUUID
} else {
objectUUID, createErr = s.store.CreatePageAtPathInDomain(payload.DomainUUID, target)
}
if createErr != nil {
s.store.FailImportSession(session.UUID, createErr)
return createErr
}
contentType := item.ContentType
filename := filepath.Base(item.SourcePath)
if item.ContentType == "text/markdown" {
body = []byte(render.Markdown(string(body)))
contentType = "text/html; charset=utf-8"
}
if !hasPrior {
if err = s.store.AddImportJournal(session.UUID, item.Sequence, target, objectUUID); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
}
if _, err = s.store.SaveRevisionContent(objectUUID, body, "archive import: "+session.ArchiveName, s.languageForDomainUUID(payload.DomainUUID), contentType, filename); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
if err = s.applyImportCityItem(session, item, target, objectUUID, cityState); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
if err = s.store.CompleteImportJournal(session.UUID, item.Sequence); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
}
s.store.MarkImportStatus(session.UUID, "completed")
return nil
}
type importCityState struct{ quartiers, streets, spots map[string]string }
func (s *Server) applyImportCityItem(session store.ImportSession, item store.ImportItem, target, pageUUID string, state *importCityState) error {
qName, streetName := strings.TrimSpace(item.Quartier), strings.TrimSpace(item.Street)
if qName == "" || streetName == "" {
return nil
}
actor := session.ActorUUID
if actor == "" {
actor = "system:import"
}
source := strings.Split(strings.Trim(item.SourcePath, "/"), "/")
if len(source) < 2 {
return nil
}
qPath := path.Join(session.TargetPrefix, source[0])
streetPath := path.Join(qPath, source[1])
qID := state.quartiers[qName]
if qID == "" {
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "quartier", "", qPath)
if err != nil {
node, err = s.store.CityNode(session.DomainUUID, "quartier", qName)
}
if err == nil {
qID = node.UUID
if node.PathPrefix == "" || node.PathPrefix == "/" {
_ = s.store.UpdateCityNodePath(session.DomainUUID, actor, "quartier", qID, qPath)
}
} else {
qID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "quartier", "", qName, qName, "", qPath)
if err != nil {
return err
}
if err = s.store.AddImportCityJournal(session.UUID, "quartier", qID); err != nil {
return err
}
}
state.quartiers[qName] = qID
}
streetKey := qID + "/" + streetName
streetID := state.streets[streetKey]
if streetID == "" {
overlap, overlapErr := s.store.CityNodeOverlappingPath(session.DomainUUID, "street", qID, streetPath)
if overlapErr == nil {
streetID = overlap.UUID
}
city, err := s.store.GetCityMap(session.DomainUUID)
if err != nil {
return err
}
if streetID == "" {
for _, q := range city.Quartiers {
if q.UUID == qID {
for _, st := range q.Streets {
if st.Name == streetName {
streetID = st.UUID
break
}
}
}
}
}
if streetID == "" {
streetID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "street", qID, streetName, streetName, "", streetPath)
if err != nil {
return err
}
if err = s.store.AddImportCityJournal(session.UUID, "street", streetID); err != nil {
return err
}
}
state.streets[streetKey] = streetID
}
spotName := strings.TrimSpace(item.Spot)
spotTitle := spotName
spotPath := path.Join(session.TargetPrefix, path.Dir(item.SourcePath))
if spotName == "" || spotName == streetName {
spotName = streetName + "-undefined"
spotTitle = "Undefined"
spotPath = path.Join(streetPath, "_undefined")
} else if len(source) <= 3 {
// A named Spot for a Page immediately below its Street has no natural
// directory boundary. Give it an explicit synthetic child boundary so
// it remains a strict descendant while the Page is linked explicitly.
spotPath = path.Join(streetPath, "_"+spotName)
}
spotKey := streetID + "/" + spotName
spotID := state.spots[spotKey]
if spotID == "" {
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "spot", streetID, spotPath)
if err != nil {
node, err = s.store.CityNode(session.DomainUUID, "spot", spotName)
}
if err == nil {
spotID = node.UUID
} else {
spotID, err = s.store.CreateSpotAtPath(session.DomainUUID, actor, streetID, spotName, spotTitle, "Imported without a defined Spot", spotPath)
if err != nil {
return err
}
if err = s.store.AddImportCityJournal(session.UUID, "spot", spotID); err != nil {
return err
}
}
state.spots[spotKey] = spotID
}
if err := s.store.AddSpotPage(session.DomainUUID, actor, spotID, pageUUID, target, "primary", item.Title); err != nil {
return err
}
return s.store.AddImportCityEdge(session.UUID, spotID, pageUUID)
}
func nextImportPath(st *store.Store, domainUUID, target string) string {
ext := path.Ext(target)
base := strings.TrimSuffix(target, ext)
for i := 2; ; i++ {
candidate := base + "-" + strconv.Itoa(i) + ext
if _, err := st.ResolvePathInDomain(domainUUID, candidate); err != nil {
return candidate
}
}
}
func (s *Server) languageForDomainUUID(domainUUID string) string {
domain, err := s.store.GetDomainByUUID(domainUUID)
if err != nil {
return "und"
}
campus := s.campusForDomain(domain)
if campus == nil || strings.TrimSpace(campus.DefaultLang) == "" {
return "und"
}
return campus.DefaultLang
}
func (s *Server) rollbackImport(domainUUID, sessionUUID string) error {
session, err := s.store.ImportSession(domainUUID, sessionUUID)
if err != nil {
return err
}
if session.Status != "completed" && session.Status != "failed" {
return errors.New("only a completed or failed import can be rolled back")
}
entries, err := s.store.ImportJournal(sessionUUID)
if err != nil {
return err
}
actor := session.ActorUUID
if actor == "" {
actor = "system:import"
}
edges, err := s.store.ImportCityEdges(sessionUUID)
if err != nil {
return err
}
for _, edge := range edges {
_ = s.store.RemoveSpotPage(domainUUID, actor, edge.SpotUUID, edge.PageUUID, "primary")
}
nodes, err := s.store.ImportCityJournal(sessionUUID)
if err != nil {
return err
}
for _, node := range nodes {
_ = s.store.DeleteCityNode(domainUUID, actor, node.Kind, node.UUID)
}
tx, err := s.store.DB.Begin()
if err != nil {
return err
}
defer tx.Rollback()
for _, x := range entries {
if _, err = tx.Exec(`DELETE FROM paths WHERE domain_uuid=? AND path=? AND object_uuid=?`, domainUUID, x.Path, x.ObjectUUID); err != nil {
return err
}
}
if _, err = tx.Exec(`UPDATE import_sessions SET status='rolled_back',updated_at_ms=? WHERE uuid=?`, time.Now().UnixMilli(), sessionUUID); err != nil {
return err
}
if err = tx.Commit(); err == nil {
s.store.InvalidatePathCaches()
}
return err
}
func readImportArchive(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
if strings.HasSuffix(strings.ToLower(name), ".zip") {
r, err := zip.OpenReader(filename)
if err != nil {
return nil, err
}
defer r.Close()
out := map[string][]byte{}
for _, f := range r.File {
clean, err := safeArchiveName(f.Name)
if err != nil {
return nil, err
}
if !wanted[clean] {
continue
}
rc, err := f.Open()
if err != nil {
return nil, err
}
data, err := io.ReadAll(io.LimitReader(rc, maxImportExpandedBytes+1))
rc.Close()
if err != nil {
return nil, err
}
out[clean] = data
}
return out, nil
}
return readImportTar(filename, name, wanted)
}
func readImportTar(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
var reader io.Reader = file
closers := []io.Closer{file}
lower := strings.ToLower(name)
var cmd *exec.Cmd
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
gz, e := gzip.NewReader(file)
if e != nil {
return nil, e
}
reader = gz
closers = append([]io.Closer{gz}, closers...)
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
file.Close()
cmd = exec.Command("xz", "-dc", filename)
pipe, e := cmd.StdoutPipe()
if e != nil {
return nil, e
}
if e = cmd.Start(); e != nil {
return nil, e
}
reader = pipe
closers = []io.Closer{pipe}
}
defer func() {
for _, c := range closers {
c.Close()
}
if cmd != nil {
cmd.Wait()
}
}()
out := map[string][]byte{}
tr := tar.NewReader(reader)
for {
h, e := tr.Next()
if e == io.EOF {
break
}
if e != nil {
return nil, e
}
clean, e := safeArchiveName(h.Name)
if e != nil {
return nil, e
}
if h.Typeflag == tar.TypeReg && wanted[clean] {
data, e := io.ReadAll(io.LimitReader(tr, maxImportExpandedBytes+1))
if e != nil {
return nil, e
}
out[clean] = data
}
}
return out, nil
}
var suggestionNameRE = regexp.MustCompile(`[^a-z0-9_-]+`)
func citySuggestionName(v string) string {
v = strings.ToLower(v)
v = suggestionNameRE.ReplaceAllString(v, "-")
return strings.Trim(v, "-")
}
func scanImportZIP(filename, target string) ([]store.ImportItem, error) {
r, err := zip.OpenReader(filename)
if err != nil {
return nil, err
}
defer r.Close()
var items []store.ImportItem
var total int64
for _, f := range r.File {
if f.FileInfo().Mode()&os.ModeSymlink != 0 {
return nil, errors.New("symbolic links are not allowed")
}
total += int64(f.UncompressedSize64)
if total > maxImportExpandedBytes {
return nil, errors.New("expanded archive is too large")
}
if !f.FileInfo().IsDir() {
if err := appendImportItem(&items, f.Name, int64(f.UncompressedSize64), target); err != nil {
return nil, err
}
}
}
return items, nil
}
func scanImportTar(filename, archiveName, target string) ([]store.ImportItem, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
var reader io.Reader = file
closers := []io.Closer{file}
lower := strings.ToLower(archiveName)
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
gz, e := gzip.NewReader(file)
if e != nil {
file.Close()
return nil, e
}
reader = gz
closers = append([]io.Closer{gz}, closers...)
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
_ = file.Close()
cmd := exec.Command("xz", "-dc", filename)
pipe, e := cmd.StdoutPipe()
if e != nil {
return nil, e
}
if e = cmd.Start(); e != nil {
return nil, e
}
reader = pipe
closers = []io.Closer{pipe}
defer cmd.Wait()
}
defer func() {
for _, c := range closers {
_ = c.Close()
}
}()
tr := tar.NewReader(reader)
var items []store.ImportItem
var total int64
for {
h, e := tr.Next()
if e == io.EOF {
break
}
if e != nil {
return nil, e
}
if h.Typeflag == tar.TypeSymlink || h.Typeflag == tar.TypeLink {
return nil, errors.New("archive links are not allowed")
}
if h.Typeflag != tar.TypeReg {
continue
}
total += h.Size
if total > maxImportExpandedBytes {
return nil, errors.New("expanded archive is too large")
}
if err := appendImportItem(&items, h.Name, h.Size, target); err != nil {
return nil, err
}
}
return items, nil
}