Files
pages/internal/server/import_handler.go
T
2026-11-15 05:47:14 +09:00

814 lines
24 KiB
Go

package server
import (
"archive/tar"
"archive/zip"
"compress/gzip"
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"pageswiki/internal/render"
"pageswiki/internal/store"
)
const importAnalyzeEventType = "import.analyze"
const importCommitEventType = "import.commit"
const maxImportArchiveBytes = int64(512 << 20)
const maxImportFiles = 10000
const maxImportExpandedBytes = int64(2 << 30)
type importAnalyzePayload struct{ DomainUUID, SessionUUID string }
type importCommitPayload struct{ DomainUUID, SessionUUID, Conflict string }
func (s *Server) initializeImportRoot() {
base := filepath.Dir(s.store.DocumentRoot)
if s.store.DocumentRoot == "" {
base = "data"
}
s.importRoot = filepath.Join(base, "imports")
if err := os.MkdirAll(s.importRoot, 0700); err != nil {
s.importRoot = ""
}
}
func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
domainUUID := s.domainUUIDForRequest(r)
if r.Method == http.MethodPost {
s.handleImportUpload(w, r, domainUUID)
return
}
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/v/import"), "/")
data := map[string]any{"Title": translate(s.getLocale(domainUUID), "import.title"), "Locale": s.getLocale(domainUUID)}
if id == "" {
sessions, err := s.store.ListImportSessions(domainUUID)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
data["Sessions"] = sessions
} else {
session, err := s.store.ImportSession(domainUUID, id)
if err != nil {
http.NotFound(w, r)
return
}
items, err := s.store.ListImportItems(id)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
s.mergeExistingCitySuggestions(domainUUID, session.TargetPrefix, items)
if session.Status == "failed" {
pending := items[:0]
for _, item := range items {
if !item.Imported {
pending = append(pending, item)
}
}
items = pending
}
data["Session"], data["Items"] = session, items
}
s.renderWithContext(w, r, "import.gohtml", data)
}
func (s *Server) handleImportAPI(w http.ResponseWriter, r *http.Request) {
domainUUID := s.domainUUIDForRequest(r)
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/_/api/v1/import/"), "/")
if id == "" {
http.NotFound(w, r)
return
}
session, err := s.store.ImportSession(domainUUID, id)
if err != nil {
http.NotFound(w, r)
return
}
if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" {
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
if authErr != nil || (session.ActorUUID != principal.UUID && !s.store.IsBuiltinGroupAdmin(domainUUID, principal.UUID, "domain_admin")) {
writeEventJSON(w, http.StatusForbidden, map[string]string{"error": "import owner or domain admin required"})
return
}
}
if r.Method == http.MethodPatch {
var request struct {
Items []store.ImportItem `json:"items"`
}
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&request) != nil {
writeEventJSON(w, 400, map[string]string{"error": "invalid JSON"})
return
}
if err := s.store.UpdateImportPlan(domainUUID, id, request.Items); err != nil {
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
return
}
writeEventJSON(w, 200, map[string]string{"status": "ok"})
return
}
if r.Method == http.MethodPost {
var request struct {
Conflict string `json:"conflict"`
}
_ = json.NewDecoder(r.Body).Decode(&request)
if request.Conflict != "rename" {
request.Conflict = "skip"
}
if err := s.store.QueueImportCommit(domainUUID, id); err != nil {
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
return
}
payload, _ := json.Marshal(importCommitPayload{DomainUUID: domainUUID, SessionUUID: id, Conflict: request.Conflict})
queued, err := s.store.EnqueueEvent(importCommitEventType, id, string(payload), time.Now())
if err != nil || !queued {
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
writeEventJSON(w, 500, map[string]string{"error": "could not queue import"})
return
}
s.wakeEventScheduler()
writeEventJSON(w, 202, map[string]string{"status": "queued"})
return
}
if r.Method == http.MethodDelete {
if err := s.rollbackImport(domainUUID, id); err != nil {
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
return
}
writeEventJSON(w, 200, map[string]string{"status": "rolled_back"})
return
}
w.WriteHeader(405)
}
func (s *Server) handleImportUpload(w http.ResponseWriter, r *http.Request, domainUUID string) {
if s.importRoot == "" {
http.Error(w, "import storage unavailable", 500)
return
}
r.Body = http.MaxBytesReader(w, r.Body, maxImportArchiveBytes+(1<<20))
if err := r.ParseMultipartForm(32 << 20); err != nil {
http.Error(w, err.Error(), 413)
return
}
file, header, err := r.FormFile("archive")
if err != nil {
http.Error(w, "archive is required", 400)
return
}
defer file.Close()
name := filepath.Base(header.Filename)
if !supportedImportArchive(name) {
http.Error(w, "supported archives: .zip, .tar, .tar.gz, .tgz, .tar.xz, .txz", 400)
return
}
target := normalizeImportTarget(r.FormValue("target_prefix"))
actor := ""
mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode")
if mode == "enforcing" {
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
if authErr != nil || s.store.Authorize(store.AuthorizationRequest{DomainUUID: domainUUID, PrincipalUUID: principal.UUID, Capability: "upload", TargetType: "path", TargetID: target}) != nil {
http.Error(w, "forbidden", 403)
return
}
actor = principal.UUID
}
temp, err := os.CreateTemp(s.importRoot, "upload-*")
if err != nil {
http.Error(w, err.Error(), 500)
return
}
tempPath := temp.Name()
defer func() { _ = temp.Close() }()
n, err := io.Copy(temp, io.LimitReader(file, maxImportArchiveBytes+1))
if err != nil || n > maxImportArchiveBytes {
_ = os.Remove(tempPath)
http.Error(w, "archive is too large or unreadable", 413)
return
}
_ = temp.Close()
id, err := s.store.CreateImportSession(domainUUID, actor, name, tempPath, target)
if err != nil {
_ = os.Remove(tempPath)
http.Error(w, err.Error(), 500)
return
}
payload, _ := json.Marshal(importAnalyzePayload{DomainUUID: domainUUID, SessionUUID: id})
queued, err := s.store.EnqueueEvent(importAnalyzeEventType, id, string(payload), time.Now())
if err != nil || !queued {
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
http.Error(w, "could not queue import", 500)
return
}
s.wakeEventScheduler()
http.Redirect(w, r, "/v/import/"+id, 303)
}
func supportedImportArchive(name string) bool {
name = strings.ToLower(name)
for _, suffix := range []string{".zip", ".tar", ".tar.gz", ".tgz", ".tar.xz", ".txz"} {
if strings.HasSuffix(name, suffix) {
return true
}
}
return false
}
func normalizeImportTarget(value string) string {
value = "/" + strings.Trim(strings.TrimSpace(value), "/")
if value == "/" {
return value
}
return path.Clean(value)
}
func safeArchiveName(name string) (string, error) {
name = strings.ReplaceAll(name, "\\", "/")
if strings.HasPrefix(name, "/") || strings.ContainsRune(name, '\x00') {
return "", errors.New("unsafe archive path")
}
for _, part := range strings.Split(name, "/") {
if part == ".." {
return "", errors.New("unsafe archive path")
}
}
clean := path.Clean(name)
if clean == "" || clean == "." || strings.HasPrefix(clean, "../") {
return "", errors.New("unsafe archive path")
}
return clean, nil
}
func (s *Server) analyzeImport(payload importAnalyzePayload) error {
s.store.MarkImportAnalyzing(payload.SessionUUID)
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
if err != nil {
return err
}
items, err := scanImportArchive(session.ArchivePath, session.ArchiveName, session.TargetPrefix)
if err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
s.mergeExistingCitySuggestions(session.DomainUUID, session.TargetPrefix, items)
if err := s.store.ReplaceImportItems(session.UUID, items); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
return nil
}
func (s *Server) mergeExistingCitySuggestions(domainUUID, targetPrefix string, items []store.ImportItem) {
for i := range items {
source := strings.Split(strings.Trim(items[i].SourcePath, "/"), "/")
if len(source) < 2 {
continue
}
qPath := path.Join(targetPrefix, source[0])
q, err := s.store.CityNodeOverlappingPath(domainUUID, "quartier", "", qPath)
if err != nil {
q, err = s.store.CityNode(domainUUID, "quartier", items[i].Quartier)
}
if err != nil {
continue
}
items[i].Quartier = q.Name
streetPath := path.Join(qPath, source[1])
street, err := s.store.CityNodeOverlappingPath(domainUUID, "street", q.UUID, streetPath)
if err != nil {
continue
}
items[i].Street = street.Name
spotName := strings.TrimSpace(items[i].Spot)
spotPath := path.Join(targetPrefix, path.Dir(items[i].SourcePath))
if spotName == "" || spotName == street.Name {
spotPath = path.Join(streetPath, "_undefined")
} else if len(source) <= 3 {
spotPath = path.Join(streetPath, "_"+spotName)
}
spot, err := s.store.CityNodeOverlappingPath(domainUUID, "spot", street.UUID, spotPath)
if err == nil && spot.Name != street.Name+"-undefined" {
items[i].Spot = spot.Name
}
}
}
func scanImportArchive(filename, archiveName, target string) ([]store.ImportItem, error) {
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
return scanImportZIP(filename, target)
}
return scanImportTar(filename, archiveName, target)
}
func appendImportItem(items *[]store.ImportItem, name string, size int64, target string) error {
clean, err := safeArchiveName(name)
if err != nil {
return err
}
if strings.HasSuffix(clean, "/") {
return nil
}
if len(*items) >= maxImportFiles {
return errors.New("archive contains too many files")
}
parts := strings.Split(clean, "/")
title := ""
media := mime.TypeByExtension(strings.ToLower(filepath.Ext(clean)))
if strings.EqualFold(filepath.Ext(clean), ".md") {
media = "text/markdown"
title = strings.TrimSuffix(parts[len(parts)-1], filepath.Ext(clean))
}
if media == "" {
media = "application/octet-stream"
}
labels := parts
if len(labels) > 0 {
labels = labels[:len(labels)-1]
}
q, street, spot := "", "", ""
if len(labels) > 0 {
q = citySuggestionName(labels[0])
}
if len(labels) > 1 {
street = citySuggestionName(labels[1])
}
if len(labels) > 2 {
spot = citySuggestionName(labels[len(labels)-1])
}
confidence := 0.45
if media == "text/markdown" {
confidence = .7
}
targetPath := path.Join(target, clean)
if media == "text/markdown" {
targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath))
}
*items = append(*items, store.ImportItem{Sequence: len(*items) + 1, SourcePath: clean, TargetPath: targetPath, Size: size, ContentType: media, Title: title, Quartier: q, Street: street, Spot: spot, Confidence: confidence, Selected: true})
return nil
}
func (s *Server) commitImport(payload importCommitPayload) error {
s.store.MarkImportStatus(payload.SessionUUID, "importing")
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
if err != nil {
return err
}
items, err := s.store.ListImportItems(payload.SessionUUID)
if err != nil {
return err
}
wanted := map[string]bool{}
for _, x := range items {
if x.Selected && !x.Imported {
wanted[x.SourcePath] = true
}
}
journalEntries, err := s.store.ImportJournal(payload.SessionUUID)
if err != nil {
return err
}
journal := map[int]store.ImportJournalEntry{}
for _, entry := range journalEntries {
journal[entry.Sequence] = entry
}
contents, err := readImportArchive(session.ArchivePath, session.ArchiveName, wanted)
if err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
cityState := &importCityState{quartiers: map[string]string{}, streets: map[string]string{}, spots: map[string]string{}}
for _, item := range items {
if !item.Selected || item.Imported {
continue
}
body, ok := contents[item.SourcePath]
if !ok {
err = fmt.Errorf("archive entry disappeared: %s", item.SourcePath)
s.store.FailImportSession(session.UUID, err)
return err
}
target := item.TargetPath
if item.ContentType == "text/markdown" {
target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown")
}
prior, hasPrior := journal[item.Sequence]
if hasPrior {
target = prior.Path
}
if !hasPrior {
if _, resolveErr := s.store.ResolvePathInDomain(payload.DomainUUID, target); resolveErr == nil {
if payload.Conflict == "skip" {
continue
}
target = nextImportPath(s.store, payload.DomainUUID, target)
}
}
objectUUID := ""
var createErr error
if hasPrior {
objectUUID = prior.ObjectUUID
} else {
objectUUID, createErr = s.store.CreatePageAtPathInDomain(payload.DomainUUID, target)
}
if createErr != nil {
s.store.FailImportSession(session.UUID, createErr)
return createErr
}
contentType := item.ContentType
filename := filepath.Base(item.SourcePath)
if item.ContentType == "text/markdown" {
body = []byte(render.Markdown(string(body)))
contentType = "text/html; charset=utf-8"
}
if !hasPrior {
if err = s.store.AddImportJournal(session.UUID, item.Sequence, target, objectUUID); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
}
if _, err = s.store.SaveRevisionContent(objectUUID, body, "archive import: "+session.ArchiveName, s.languageForDomainUUID(payload.DomainUUID), contentType, filename); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
if err = s.applyImportCityItem(session, item, target, objectUUID, cityState); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
if err = s.store.CompleteImportJournal(session.UUID, item.Sequence); err != nil {
s.store.FailImportSession(session.UUID, err)
return err
}
}
s.store.MarkImportStatus(session.UUID, "completed")
return nil
}
type importCityState struct{ quartiers, streets, spots map[string]string }
func (s *Server) applyImportCityItem(session store.ImportSession, item store.ImportItem, target, pageUUID string, state *importCityState) error {
qName, streetName := strings.TrimSpace(item.Quartier), strings.TrimSpace(item.Street)
if qName == "" || streetName == "" {
return nil
}
actor := session.ActorUUID
if actor == "" {
actor = "system:import"
}
source := strings.Split(strings.Trim(item.SourcePath, "/"), "/")
if len(source) < 2 {
return nil
}
qPath := path.Join(session.TargetPrefix, source[0])
streetPath := path.Join(qPath, source[1])
qID := state.quartiers[qName]
if qID == "" {
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "quartier", "", qPath)
if err != nil {
node, err = s.store.CityNode(session.DomainUUID, "quartier", qName)
}
if err == nil {
qID = node.UUID
if node.PathPrefix == "" || node.PathPrefix == "/" {
_ = s.store.UpdateCityNodePath(session.DomainUUID, actor, "quartier", qID, qPath)
}
} else {
qID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "quartier", "", qName, qName, "", qPath)
if err != nil {
return err
}
if err = s.store.AddImportCityJournal(session.UUID, "quartier", qID); err != nil {
return err
}
}
state.quartiers[qName] = qID
}
streetKey := qID + "/" + streetName
streetID := state.streets[streetKey]
if streetID == "" {
overlap, overlapErr := s.store.CityNodeOverlappingPath(session.DomainUUID, "street", qID, streetPath)
if overlapErr == nil {
streetID = overlap.UUID
}
city, err := s.store.GetCityMap(session.DomainUUID)
if err != nil {
return err
}
if streetID == "" {
for _, q := range city.Quartiers {
if q.UUID == qID {
for _, st := range q.Streets {
if st.Name == streetName {
streetID = st.UUID
break
}
}
}
}
}
if streetID == "" {
streetID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "street", qID, streetName, streetName, "", streetPath)
if err != nil {
return err
}
if err = s.store.AddImportCityJournal(session.UUID, "street", streetID); err != nil {
return err
}
}
state.streets[streetKey] = streetID
}
spotName := strings.TrimSpace(item.Spot)
spotTitle := spotName
spotPath := path.Join(session.TargetPrefix, path.Dir(item.SourcePath))
if spotName == "" || spotName == streetName {
spotName = streetName + "-undefined"
spotTitle = "Undefined"
spotPath = path.Join(streetPath, "_undefined")
} else if len(source) <= 3 {
// A named Spot for a Page immediately below its Street has no natural
// directory boundary. Give it an explicit synthetic child boundary so
// it remains a strict descendant while the Page is linked explicitly.
spotPath = path.Join(streetPath, "_"+spotName)
}
spotKey := streetID + "/" + spotName
spotID := state.spots[spotKey]
if spotID == "" {
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "spot", streetID, spotPath)
if err != nil {
node, err = s.store.CityNode(session.DomainUUID, "spot", spotName)
}
if err == nil {
spotID = node.UUID
} else {
spotID, err = s.store.CreateSpotAtPath(session.DomainUUID, actor, streetID, spotName, spotTitle, "Imported without a defined Spot", spotPath)
if err != nil {
return err
}
if err = s.store.AddImportCityJournal(session.UUID, "spot", spotID); err != nil {
return err
}
}
state.spots[spotKey] = spotID
}
if err := s.store.AddSpotPage(session.DomainUUID, actor, spotID, pageUUID, target, "primary", item.Title); err != nil {
return err
}
return s.store.AddImportCityEdge(session.UUID, spotID, pageUUID)
}
func nextImportPath(st *store.Store, domainUUID, target string) string {
ext := path.Ext(target)
base := strings.TrimSuffix(target, ext)
for i := 2; ; i++ {
candidate := base + "-" + strconv.Itoa(i) + ext
if _, err := st.ResolvePathInDomain(domainUUID, candidate); err != nil {
return candidate
}
}
}
func (s *Server) languageForDomainUUID(domainUUID string) string {
domain, err := s.store.GetDomainByUUID(domainUUID)
if err != nil {
return "und"
}
campus := s.campusForDomain(domain)
if campus == nil || strings.TrimSpace(campus.DefaultLang) == "" {
return "und"
}
return campus.DefaultLang
}
func (s *Server) rollbackImport(domainUUID, sessionUUID string) error {
session, err := s.store.ImportSession(domainUUID, sessionUUID)
if err != nil {
return err
}
if session.Status != "completed" && session.Status != "failed" {
return errors.New("only a completed or failed import can be rolled back")
}
actor := session.ActorUUID
if actor == "" {
actor = "system:import"
}
edges, err := s.store.ImportCityEdges(sessionUUID)
if err != nil {
return err
}
for _, edge := range edges {
if err := s.store.RemoveSpotPage(domainUUID, actor, edge.SpotUUID, edge.PageUUID, "primary"); err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
}
nodes, err := s.store.ImportCityJournal(sessionUUID)
if err != nil {
return err
}
for _, node := range nodes {
if err := s.store.DeleteCityNode(domainUUID, actor, node.Kind, node.UUID); err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
}
return s.store.RollbackImportPaths(domainUUID, sessionUUID)
}
func readImportArchive(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
if strings.HasSuffix(strings.ToLower(name), ".zip") {
r, err := zip.OpenReader(filename)
if err != nil {
return nil, err
}
defer r.Close()
out := map[string][]byte{}
for _, f := range r.File {
clean, err := safeArchiveName(f.Name)
if err != nil {
return nil, err
}
if !wanted[clean] {
continue
}
rc, err := f.Open()
if err != nil {
return nil, err
}
data, err := io.ReadAll(io.LimitReader(rc, maxImportExpandedBytes+1))
rc.Close()
if err != nil {
return nil, err
}
out[clean] = data
}
return out, nil
}
return readImportTar(filename, name, wanted)
}
func readImportTar(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
var reader io.Reader = file
closers := []io.Closer{file}
lower := strings.ToLower(name)
var cmd *exec.Cmd
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
gz, e := gzip.NewReader(file)
if e != nil {
return nil, e
}
reader = gz
closers = append([]io.Closer{gz}, closers...)
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
file.Close()
cmd = exec.Command("xz", "-dc", filename)
pipe, e := cmd.StdoutPipe()
if e != nil {
return nil, e
}
if e = cmd.Start(); e != nil {
return nil, e
}
reader = pipe
closers = []io.Closer{pipe}
}
defer func() {
for _, c := range closers {
c.Close()
}
if cmd != nil {
cmd.Wait()
}
}()
out := map[string][]byte{}
tr := tar.NewReader(reader)
for {
h, e := tr.Next()
if e == io.EOF {
break
}
if e != nil {
return nil, e
}
clean, e := safeArchiveName(h.Name)
if e != nil {
return nil, e
}
if h.Typeflag == tar.TypeReg && wanted[clean] {
data, e := io.ReadAll(io.LimitReader(tr, maxImportExpandedBytes+1))
if e != nil {
return nil, e
}
out[clean] = data
}
}
return out, nil
}
var suggestionNameRE = regexp.MustCompile(`[^a-z0-9_-]+`)
func citySuggestionName(v string) string {
v = strings.ToLower(v)
v = suggestionNameRE.ReplaceAllString(v, "-")
return strings.Trim(v, "-")
}
func scanImportZIP(filename, target string) ([]store.ImportItem, error) {
r, err := zip.OpenReader(filename)
if err != nil {
return nil, err
}
defer r.Close()
var items []store.ImportItem
var total int64
for _, f := range r.File {
if f.FileInfo().Mode()&os.ModeSymlink != 0 {
return nil, errors.New("symbolic links are not allowed")
}
total += int64(f.UncompressedSize64)
if total > maxImportExpandedBytes {
return nil, errors.New("expanded archive is too large")
}
if !f.FileInfo().IsDir() {
if err := appendImportItem(&items, f.Name, int64(f.UncompressedSize64), target); err != nil {
return nil, err
}
}
}
return items, nil
}
func scanImportTar(filename, archiveName, target string) ([]store.ImportItem, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
var reader io.Reader = file
closers := []io.Closer{file}
lower := strings.ToLower(archiveName)
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
gz, e := gzip.NewReader(file)
if e != nil {
file.Close()
return nil, e
}
reader = gz
closers = append([]io.Closer{gz}, closers...)
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
_ = file.Close()
cmd := exec.Command("xz", "-dc", filename)
pipe, e := cmd.StdoutPipe()
if e != nil {
return nil, e
}
if e = cmd.Start(); e != nil {
return nil, e
}
reader = pipe
closers = []io.Closer{pipe}
defer cmd.Wait()
}
defer func() {
for _, c := range closers {
_ = c.Close()
}
}()
tr := tar.NewReader(reader)
var items []store.ImportItem
var total int64
for {
h, e := tr.Next()
if e == io.EOF {
break
}
if e != nil {
return nil, e
}
if h.Typeflag == tar.TypeSymlink || h.Typeflag == tar.TypeLink {
return nil, errors.New("archive links are not allowed")
}
if h.Typeflag != tar.TypeReg {
continue
}
total += h.Size
if total > maxImportExpandedBytes {
return nil, errors.New("expanded archive is too large")
}
if err := appendImportItem(&items, h.Name, h.Size, target); err != nil {
return nil, err
}
}
return items, nil
}