package server import ( "archive/tar" "archive/zip" "compress/gzip" "encoding/json" "errors" "fmt" "io" "mime" "net/http" "os" "os/exec" "path" "path/filepath" "regexp" "strconv" "strings" "time" "pageswiki/internal/render" "pageswiki/internal/store" ) const importAnalyzeEventType = "import.analyze" const importCommitEventType = "import.commit" const maxImportArchiveBytes = int64(512 << 20) const maxImportFiles = 10000 const maxImportExpandedBytes = int64(2 << 30) type importAnalyzePayload struct{ DomainUUID, SessionUUID string } type importCommitPayload struct{ DomainUUID, SessionUUID, Conflict string } func (s *Server) initializeImportRoot() { base := filepath.Dir(s.store.DocumentRoot) if s.store.DocumentRoot == "" { base = "data" } s.importRoot = filepath.Join(base, "imports") if err := os.MkdirAll(s.importRoot, 0700); err != nil { s.importRoot = "" } } func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) { domainUUID := s.domainUUIDForRequest(r) if r.Method == http.MethodPost { s.handleImportUpload(w, r, domainUUID) return } if r.Method != http.MethodGet { http.Error(w, "method not allowed", 405) return } id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/v/import"), "/") data := map[string]any{"Title": translate(s.getLocale(domainUUID), "import.title"), "Locale": s.getLocale(domainUUID)} if id == "" { sessions, err := s.store.ListImportSessions(domainUUID) if err != nil { http.Error(w, err.Error(), 500) return } data["Sessions"] = sessions } else { session, err := s.store.ImportSession(domainUUID, id) if err != nil { http.NotFound(w, r) return } items, err := s.store.ListImportItems(id) if err != nil { http.Error(w, err.Error(), 500) return } s.mergeExistingCitySuggestions(domainUUID, session.TargetPrefix, items) if session.Status == "failed" { pending := items[:0] for _, item := range items { if !item.Imported { pending = append(pending, item) } } items = pending } data["Session"], data["Items"] = session, items } s.renderWithContext(w, r, "import.gohtml", data) } func (s *Server) handleImportAPI(w http.ResponseWriter, r *http.Request) { domainUUID := s.domainUUIDForRequest(r) id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/_/api/v1/import/"), "/") if id == "" { http.NotFound(w, r) return } session, err := s.store.ImportSession(domainUUID, id) if err != nil { http.NotFound(w, r) return } if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" { principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r)) if authErr != nil || (session.ActorUUID != principal.UUID && !s.store.IsBuiltinGroupAdmin(domainUUID, principal.UUID, "domain_admin")) { writeEventJSON(w, http.StatusForbidden, map[string]string{"error": "import owner or domain admin required"}) return } } if r.Method == http.MethodPatch { var request struct { Items []store.ImportItem `json:"items"` } if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&request) != nil { writeEventJSON(w, 400, map[string]string{"error": "invalid JSON"}) return } if err := s.store.UpdateImportPlan(domainUUID, id, request.Items); err != nil { writeEventJSON(w, 400, map[string]string{"error": err.Error()}) return } writeEventJSON(w, 200, map[string]string{"status": "ok"}) return } if r.Method == http.MethodPost { var request struct { Conflict string `json:"conflict"` } _ = json.NewDecoder(r.Body).Decode(&request) if request.Conflict != "rename" { request.Conflict = "skip" } if err := s.store.QueueImportCommit(domainUUID, id); err != nil { writeEventJSON(w, 400, map[string]string{"error": err.Error()}) return } payload, _ := json.Marshal(importCommitPayload{DomainUUID: domainUUID, SessionUUID: id, Conflict: request.Conflict}) queued, err := s.store.EnqueueEvent(importCommitEventType, id, string(payload), time.Now()) if err != nil || !queued { s.store.FailImportSession(id, fmt.Errorf("queue: %v", err)) writeEventJSON(w, 500, map[string]string{"error": "could not queue import"}) return } s.wakeEventScheduler() writeEventJSON(w, 202, map[string]string{"status": "queued"}) return } if r.Method == http.MethodDelete { if err := s.rollbackImport(domainUUID, id); err != nil { writeEventJSON(w, 400, map[string]string{"error": err.Error()}) return } writeEventJSON(w, 200, map[string]string{"status": "rolled_back"}) return } w.WriteHeader(405) } func (s *Server) handleImportUpload(w http.ResponseWriter, r *http.Request, domainUUID string) { if s.importRoot == "" { http.Error(w, "import storage unavailable", 500) return } r.Body = http.MaxBytesReader(w, r.Body, maxImportArchiveBytes+(1<<20)) if err := r.ParseMultipartForm(32 << 20); err != nil { http.Error(w, err.Error(), 413) return } file, header, err := r.FormFile("archive") if err != nil { http.Error(w, "archive is required", 400) return } defer file.Close() name := filepath.Base(header.Filename) if !supportedImportArchive(name) { http.Error(w, "supported archives: .zip, .tar, .tar.gz, .tgz, .tar.xz, .txz", 400) return } target := normalizeImportTarget(r.FormValue("target_prefix")) actor := "" mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode") if mode == "enforcing" { principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r)) if authErr != nil || s.store.Authorize(store.AuthorizationRequest{DomainUUID: domainUUID, PrincipalUUID: principal.UUID, Capability: "upload", TargetType: "path", TargetID: target}) != nil { http.Error(w, "forbidden", 403) return } actor = principal.UUID } temp, err := os.CreateTemp(s.importRoot, "upload-*") if err != nil { http.Error(w, err.Error(), 500) return } tempPath := temp.Name() defer func() { _ = temp.Close() }() n, err := io.Copy(temp, io.LimitReader(file, maxImportArchiveBytes+1)) if err != nil || n > maxImportArchiveBytes { _ = os.Remove(tempPath) http.Error(w, "archive is too large or unreadable", 413) return } _ = temp.Close() id, err := s.store.CreateImportSession(domainUUID, actor, name, tempPath, target) if err != nil { _ = os.Remove(tempPath) http.Error(w, err.Error(), 500) return } payload, _ := json.Marshal(importAnalyzePayload{DomainUUID: domainUUID, SessionUUID: id}) queued, err := s.store.EnqueueEvent(importAnalyzeEventType, id, string(payload), time.Now()) if err != nil || !queued { s.store.FailImportSession(id, fmt.Errorf("queue: %v", err)) http.Error(w, "could not queue import", 500) return } s.wakeEventScheduler() http.Redirect(w, r, "/v/import/"+id, 303) } func supportedImportArchive(name string) bool { name = strings.ToLower(name) for _, suffix := range []string{".zip", ".tar", ".tar.gz", ".tgz", ".tar.xz", ".txz"} { if strings.HasSuffix(name, suffix) { return true } } return false } func normalizeImportTarget(value string) string { value = "/" + strings.Trim(strings.TrimSpace(value), "/") if value == "/" { return value } return path.Clean(value) } func safeArchiveName(name string) (string, error) { name = strings.ReplaceAll(name, "\\", "/") if strings.HasPrefix(name, "/") || strings.ContainsRune(name, '\x00') { return "", errors.New("unsafe archive path") } for _, part := range strings.Split(name, "/") { if part == ".." { return "", errors.New("unsafe archive path") } } clean := path.Clean(name) if clean == "" || clean == "." || strings.HasPrefix(clean, "../") { return "", errors.New("unsafe archive path") } return clean, nil } func (s *Server) analyzeImport(payload importAnalyzePayload) error { s.store.MarkImportAnalyzing(payload.SessionUUID) session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID) if err != nil { return err } items, err := scanImportArchive(session.ArchivePath, session.ArchiveName, session.TargetPrefix) if err != nil { s.store.FailImportSession(session.UUID, err) return err } s.mergeExistingCitySuggestions(session.DomainUUID, session.TargetPrefix, items) if err := s.store.ReplaceImportItems(session.UUID, items); err != nil { s.store.FailImportSession(session.UUID, err) return err } return nil } func (s *Server) mergeExistingCitySuggestions(domainUUID, targetPrefix string, items []store.ImportItem) { for i := range items { source := strings.Split(strings.Trim(items[i].SourcePath, "/"), "/") if len(source) < 2 { continue } qPath := path.Join(targetPrefix, source[0]) q, err := s.store.CityNodeOverlappingPath(domainUUID, "quartier", "", qPath) if err != nil { q, err = s.store.CityNode(domainUUID, "quartier", items[i].Quartier) } if err != nil { continue } items[i].Quartier = q.Name streetPath := path.Join(qPath, source[1]) street, err := s.store.CityNodeOverlappingPath(domainUUID, "street", q.UUID, streetPath) if err != nil { continue } items[i].Street = street.Name spotName := strings.TrimSpace(items[i].Spot) spotPath := path.Join(targetPrefix, path.Dir(items[i].SourcePath)) if spotName == "" || spotName == street.Name { spotPath = path.Join(streetPath, "_undefined") } else if len(source) <= 3 { spotPath = path.Join(streetPath, "_"+spotName) } spot, err := s.store.CityNodeOverlappingPath(domainUUID, "spot", street.UUID, spotPath) if err == nil && spot.Name != street.Name+"-undefined" { items[i].Spot = spot.Name } } } func scanImportArchive(filename, archiveName, target string) ([]store.ImportItem, error) { if strings.HasSuffix(strings.ToLower(archiveName), ".zip") { return scanImportZIP(filename, target) } return scanImportTar(filename, archiveName, target) } func appendImportItem(items *[]store.ImportItem, name string, size int64, target string) error { clean, err := safeArchiveName(name) if err != nil { return err } if strings.HasSuffix(clean, "/") { return nil } if len(*items) >= maxImportFiles { return errors.New("archive contains too many files") } parts := strings.Split(clean, "/") title := "" media := mime.TypeByExtension(strings.ToLower(filepath.Ext(clean))) if strings.EqualFold(filepath.Ext(clean), ".md") { media = "text/markdown" title = strings.TrimSuffix(parts[len(parts)-1], filepath.Ext(clean)) } if media == "" { media = "application/octet-stream" } labels := parts if len(labels) > 0 { labels = labels[:len(labels)-1] } q, street, spot := "", "", "" if len(labels) > 0 { q = citySuggestionName(labels[0]) } if len(labels) > 1 { street = citySuggestionName(labels[1]) } if len(labels) > 2 { spot = citySuggestionName(labels[len(labels)-1]) } confidence := 0.45 if media == "text/markdown" { confidence = .7 } targetPath := path.Join(target, clean) if media == "text/markdown" { targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath)) } *items = append(*items, store.ImportItem{Sequence: len(*items) + 1, SourcePath: clean, TargetPath: targetPath, Size: size, ContentType: media, Title: title, Quartier: q, Street: street, Spot: spot, Confidence: confidence, Selected: true}) return nil } func (s *Server) commitImport(payload importCommitPayload) error { s.store.MarkImportStatus(payload.SessionUUID, "importing") session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID) if err != nil { return err } items, err := s.store.ListImportItems(payload.SessionUUID) if err != nil { return err } wanted := map[string]bool{} for _, x := range items { if x.Selected && !x.Imported { wanted[x.SourcePath] = true } } journalEntries, err := s.store.ImportJournal(payload.SessionUUID) if err != nil { return err } journal := map[int]store.ImportJournalEntry{} for _, entry := range journalEntries { journal[entry.Sequence] = entry } contents, err := readImportArchive(session.ArchivePath, session.ArchiveName, wanted) if err != nil { s.store.FailImportSession(session.UUID, err) return err } cityState := &importCityState{quartiers: map[string]string{}, streets: map[string]string{}, spots: map[string]string{}} for _, item := range items { if !item.Selected || item.Imported { continue } body, ok := contents[item.SourcePath] if !ok { err = fmt.Errorf("archive entry disappeared: %s", item.SourcePath) s.store.FailImportSession(session.UUID, err) return err } target := item.TargetPath if item.ContentType == "text/markdown" { target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown") } prior, hasPrior := journal[item.Sequence] if hasPrior { target = prior.Path } if !hasPrior { if _, resolveErr := s.store.ResolvePathInDomain(payload.DomainUUID, target); resolveErr == nil { if payload.Conflict == "skip" { continue } target = nextImportPath(s.store, payload.DomainUUID, target) } } objectUUID := "" var createErr error if hasPrior { objectUUID = prior.ObjectUUID } else { objectUUID, createErr = s.store.CreatePageAtPathInDomain(payload.DomainUUID, target) } if createErr != nil { s.store.FailImportSession(session.UUID, createErr) return createErr } contentType := item.ContentType filename := filepath.Base(item.SourcePath) if item.ContentType == "text/markdown" { body = []byte(render.Markdown(string(body))) contentType = "text/html; charset=utf-8" } if !hasPrior { if err = s.store.AddImportJournal(session.UUID, item.Sequence, target, objectUUID); err != nil { s.store.FailImportSession(session.UUID, err) return err } } if _, err = s.store.SaveRevisionContent(objectUUID, body, "archive import: "+session.ArchiveName, s.languageForDomainUUID(payload.DomainUUID), contentType, filename); err != nil { s.store.FailImportSession(session.UUID, err) return err } if err = s.applyImportCityItem(session, item, target, objectUUID, cityState); err != nil { s.store.FailImportSession(session.UUID, err) return err } if err = s.store.CompleteImportJournal(session.UUID, item.Sequence); err != nil { s.store.FailImportSession(session.UUID, err) return err } } s.store.MarkImportStatus(session.UUID, "completed") return nil } type importCityState struct{ quartiers, streets, spots map[string]string } func (s *Server) applyImportCityItem(session store.ImportSession, item store.ImportItem, target, pageUUID string, state *importCityState) error { qName, streetName := strings.TrimSpace(item.Quartier), strings.TrimSpace(item.Street) if qName == "" || streetName == "" { return nil } actor := session.ActorUUID if actor == "" { actor = "system:import" } source := strings.Split(strings.Trim(item.SourcePath, "/"), "/") if len(source) < 2 { return nil } qPath := path.Join(session.TargetPrefix, source[0]) streetPath := path.Join(qPath, source[1]) qID := state.quartiers[qName] if qID == "" { node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "quartier", "", qPath) if err != nil { node, err = s.store.CityNode(session.DomainUUID, "quartier", qName) } if err == nil { qID = node.UUID if node.PathPrefix == "" || node.PathPrefix == "/" { _ = s.store.UpdateCityNodePath(session.DomainUUID, actor, "quartier", qID, qPath) } } else { qID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "quartier", "", qName, qName, "", qPath) if err != nil { return err } if err = s.store.AddImportCityJournal(session.UUID, "quartier", qID); err != nil { return err } } state.quartiers[qName] = qID } streetKey := qID + "/" + streetName streetID := state.streets[streetKey] if streetID == "" { overlap, overlapErr := s.store.CityNodeOverlappingPath(session.DomainUUID, "street", qID, streetPath) if overlapErr == nil { streetID = overlap.UUID } city, err := s.store.GetCityMap(session.DomainUUID) if err != nil { return err } if streetID == "" { for _, q := range city.Quartiers { if q.UUID == qID { for _, st := range q.Streets { if st.Name == streetName { streetID = st.UUID break } } } } } if streetID == "" { streetID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "street", qID, streetName, streetName, "", streetPath) if err != nil { return err } if err = s.store.AddImportCityJournal(session.UUID, "street", streetID); err != nil { return err } } state.streets[streetKey] = streetID } spotName := strings.TrimSpace(item.Spot) spotTitle := spotName spotPath := path.Join(session.TargetPrefix, path.Dir(item.SourcePath)) if spotName == "" || spotName == streetName { spotName = streetName + "-undefined" spotTitle = "Undefined" spotPath = path.Join(streetPath, "_undefined") } else if len(source) <= 3 { // A named Spot for a Page immediately below its Street has no natural // directory boundary. Give it an explicit synthetic child boundary so // it remains a strict descendant while the Page is linked explicitly. spotPath = path.Join(streetPath, "_"+spotName) } spotKey := streetID + "/" + spotName spotID := state.spots[spotKey] if spotID == "" { node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "spot", streetID, spotPath) if err != nil { node, err = s.store.CityNode(session.DomainUUID, "spot", spotName) } if err == nil { spotID = node.UUID } else { spotID, err = s.store.CreateSpotAtPath(session.DomainUUID, actor, streetID, spotName, spotTitle, "Imported without a defined Spot", spotPath) if err != nil { return err } if err = s.store.AddImportCityJournal(session.UUID, "spot", spotID); err != nil { return err } } state.spots[spotKey] = spotID } if err := s.store.AddSpotPage(session.DomainUUID, actor, spotID, pageUUID, target, "primary", item.Title); err != nil { return err } return s.store.AddImportCityEdge(session.UUID, spotID, pageUUID) } func nextImportPath(st *store.Store, domainUUID, target string) string { ext := path.Ext(target) base := strings.TrimSuffix(target, ext) for i := 2; ; i++ { candidate := base + "-" + strconv.Itoa(i) + ext if _, err := st.ResolvePathInDomain(domainUUID, candidate); err != nil { return candidate } } } func (s *Server) languageForDomainUUID(domainUUID string) string { domain, err := s.store.GetDomainByUUID(domainUUID) if err != nil { return "und" } campus := s.campusForDomain(domain) if campus == nil || strings.TrimSpace(campus.DefaultLang) == "" { return "und" } return campus.DefaultLang } func (s *Server) rollbackImport(domainUUID, sessionUUID string) error { session, err := s.store.ImportSession(domainUUID, sessionUUID) if err != nil { return err } if session.Status != "completed" && session.Status != "failed" { return errors.New("only a completed or failed import can be rolled back") } entries, err := s.store.ImportJournal(sessionUUID) if err != nil { return err } actor := session.ActorUUID if actor == "" { actor = "system:import" } edges, err := s.store.ImportCityEdges(sessionUUID) if err != nil { return err } for _, edge := range edges { _ = s.store.RemoveSpotPage(domainUUID, actor, edge.SpotUUID, edge.PageUUID, "primary") } nodes, err := s.store.ImportCityJournal(sessionUUID) if err != nil { return err } for _, node := range nodes { _ = s.store.DeleteCityNode(domainUUID, actor, node.Kind, node.UUID) } tx, err := s.store.DB.Begin() if err != nil { return err } defer tx.Rollback() for _, x := range entries { if _, err = tx.Exec(`DELETE FROM paths WHERE domain_uuid=? AND path=? AND object_uuid=?`, domainUUID, x.Path, x.ObjectUUID); err != nil { return err } } if _, err = tx.Exec(`UPDATE import_sessions SET status='rolled_back',updated_at_ms=? WHERE uuid=?`, time.Now().UnixMilli(), sessionUUID); err != nil { return err } if err = tx.Commit(); err == nil { s.store.InvalidatePathCaches() } return err } func readImportArchive(filename, name string, wanted map[string]bool) (map[string][]byte, error) { if strings.HasSuffix(strings.ToLower(name), ".zip") { r, err := zip.OpenReader(filename) if err != nil { return nil, err } defer r.Close() out := map[string][]byte{} for _, f := range r.File { clean, err := safeArchiveName(f.Name) if err != nil { return nil, err } if !wanted[clean] { continue } rc, err := f.Open() if err != nil { return nil, err } data, err := io.ReadAll(io.LimitReader(rc, maxImportExpandedBytes+1)) rc.Close() if err != nil { return nil, err } out[clean] = data } return out, nil } return readImportTar(filename, name, wanted) } func readImportTar(filename, name string, wanted map[string]bool) (map[string][]byte, error) { file, err := os.Open(filename) if err != nil { return nil, err } var reader io.Reader = file closers := []io.Closer{file} lower := strings.ToLower(name) var cmd *exec.Cmd if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") { gz, e := gzip.NewReader(file) if e != nil { return nil, e } reader = gz closers = append([]io.Closer{gz}, closers...) } else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") { file.Close() cmd = exec.Command("xz", "-dc", filename) pipe, e := cmd.StdoutPipe() if e != nil { return nil, e } if e = cmd.Start(); e != nil { return nil, e } reader = pipe closers = []io.Closer{pipe} } defer func() { for _, c := range closers { c.Close() } if cmd != nil { cmd.Wait() } }() out := map[string][]byte{} tr := tar.NewReader(reader) for { h, e := tr.Next() if e == io.EOF { break } if e != nil { return nil, e } clean, e := safeArchiveName(h.Name) if e != nil { return nil, e } if h.Typeflag == tar.TypeReg && wanted[clean] { data, e := io.ReadAll(io.LimitReader(tr, maxImportExpandedBytes+1)) if e != nil { return nil, e } out[clean] = data } } return out, nil } var suggestionNameRE = regexp.MustCompile(`[^a-z0-9_-]+`) func citySuggestionName(v string) string { v = strings.ToLower(v) v = suggestionNameRE.ReplaceAllString(v, "-") return strings.Trim(v, "-") } func scanImportZIP(filename, target string) ([]store.ImportItem, error) { r, err := zip.OpenReader(filename) if err != nil { return nil, err } defer r.Close() var items []store.ImportItem var total int64 for _, f := range r.File { if f.FileInfo().Mode()&os.ModeSymlink != 0 { return nil, errors.New("symbolic links are not allowed") } total += int64(f.UncompressedSize64) if total > maxImportExpandedBytes { return nil, errors.New("expanded archive is too large") } if !f.FileInfo().IsDir() { if err := appendImportItem(&items, f.Name, int64(f.UncompressedSize64), target); err != nil { return nil, err } } } return items, nil } func scanImportTar(filename, archiveName, target string) ([]store.ImportItem, error) { file, err := os.Open(filename) if err != nil { return nil, err } var reader io.Reader = file closers := []io.Closer{file} lower := strings.ToLower(archiveName) if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") { gz, e := gzip.NewReader(file) if e != nil { file.Close() return nil, e } reader = gz closers = append([]io.Closer{gz}, closers...) } else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") { _ = file.Close() cmd := exec.Command("xz", "-dc", filename) pipe, e := cmd.StdoutPipe() if e != nil { return nil, e } if e = cmd.Start(); e != nil { return nil, e } reader = pipe closers = []io.Closer{pipe} defer cmd.Wait() } defer func() { for _, c := range closers { _ = c.Close() } }() tr := tar.NewReader(reader) var items []store.ImportItem var total int64 for { h, e := tr.Next() if e == io.EOF { break } if e != nil { return nil, e } if h.Typeflag == tar.TypeSymlink || h.Typeflag == tar.TypeLink { return nil, errors.New("archive links are not allowed") } if h.Typeflag != tar.TypeReg { continue } total += h.Size if total > maxImportExpandedBytes { return nil, errors.New("expanded archive is too large") } if err := appendImportItem(&items, h.Name, h.Size, target); err != nil { return nil, err } } return items, nil }