814 lines
24 KiB
Go
814 lines
24 KiB
Go
package server
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"compress/gzip"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"pageswiki/internal/render"
|
|
"pageswiki/internal/store"
|
|
)
|
|
|
|
const importAnalyzeEventType = "import.analyze"
|
|
const importCommitEventType = "import.commit"
|
|
const maxImportArchiveBytes = int64(512 << 20)
|
|
const maxImportFiles = 10000
|
|
const maxImportExpandedBytes = int64(2 << 30)
|
|
|
|
type importAnalyzePayload struct{ DomainUUID, SessionUUID string }
|
|
type importCommitPayload struct{ DomainUUID, SessionUUID, Conflict string }
|
|
|
|
func (s *Server) initializeImportRoot() {
|
|
base := filepath.Dir(s.store.DocumentRoot)
|
|
if s.store.DocumentRoot == "" {
|
|
base = "data"
|
|
}
|
|
s.importRoot = filepath.Join(base, "imports")
|
|
if err := os.MkdirAll(s.importRoot, 0700); err != nil {
|
|
s.importRoot = ""
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
|
|
domainUUID := s.domainUUIDForRequest(r)
|
|
if r.Method == http.MethodPost {
|
|
s.handleImportUpload(w, r, domainUUID)
|
|
return
|
|
}
|
|
if r.Method != http.MethodGet {
|
|
http.Error(w, "method not allowed", 405)
|
|
return
|
|
}
|
|
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/v/import"), "/")
|
|
data := map[string]any{"Title": translate(s.getLocale(domainUUID), "import.title"), "Locale": s.getLocale(domainUUID)}
|
|
if id == "" {
|
|
sessions, err := s.store.ListImportSessions(domainUUID)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
data["Sessions"] = sessions
|
|
} else {
|
|
session, err := s.store.ImportSession(domainUUID, id)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
items, err := s.store.ListImportItems(id)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
s.mergeExistingCitySuggestions(domainUUID, session.TargetPrefix, items)
|
|
if session.Status == "failed" {
|
|
pending := items[:0]
|
|
for _, item := range items {
|
|
if !item.Imported {
|
|
pending = append(pending, item)
|
|
}
|
|
}
|
|
items = pending
|
|
}
|
|
data["Session"], data["Items"] = session, items
|
|
}
|
|
s.renderWithContext(w, r, "import.gohtml", data)
|
|
}
|
|
|
|
func (s *Server) handleImportAPI(w http.ResponseWriter, r *http.Request) {
|
|
domainUUID := s.domainUUIDForRequest(r)
|
|
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/_/api/v1/import/"), "/")
|
|
if id == "" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
session, err := s.store.ImportSession(domainUUID, id)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" {
|
|
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
|
|
if authErr != nil || (session.ActorUUID != principal.UUID && !s.store.IsBuiltinGroupAdmin(domainUUID, principal.UUID, "domain_admin")) {
|
|
writeEventJSON(w, http.StatusForbidden, map[string]string{"error": "import owner or domain admin required"})
|
|
return
|
|
}
|
|
}
|
|
if r.Method == http.MethodPatch {
|
|
var request struct {
|
|
Items []store.ImportItem `json:"items"`
|
|
}
|
|
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&request) != nil {
|
|
writeEventJSON(w, 400, map[string]string{"error": "invalid JSON"})
|
|
return
|
|
}
|
|
if err := s.store.UpdateImportPlan(domainUUID, id, request.Items); err != nil {
|
|
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
writeEventJSON(w, 200, map[string]string{"status": "ok"})
|
|
return
|
|
}
|
|
if r.Method == http.MethodPost {
|
|
var request struct {
|
|
Conflict string `json:"conflict"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&request)
|
|
if request.Conflict != "rename" {
|
|
request.Conflict = "skip"
|
|
}
|
|
if err := s.store.QueueImportCommit(domainUUID, id); err != nil {
|
|
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
payload, _ := json.Marshal(importCommitPayload{DomainUUID: domainUUID, SessionUUID: id, Conflict: request.Conflict})
|
|
queued, err := s.store.EnqueueEvent(importCommitEventType, id, string(payload), time.Now())
|
|
if err != nil || !queued {
|
|
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
|
|
writeEventJSON(w, 500, map[string]string{"error": "could not queue import"})
|
|
return
|
|
}
|
|
s.wakeEventScheduler()
|
|
writeEventJSON(w, 202, map[string]string{"status": "queued"})
|
|
return
|
|
}
|
|
if r.Method == http.MethodDelete {
|
|
if err := s.rollbackImport(domainUUID, id); err != nil {
|
|
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
writeEventJSON(w, 200, map[string]string{"status": "rolled_back"})
|
|
return
|
|
}
|
|
w.WriteHeader(405)
|
|
}
|
|
|
|
func (s *Server) handleImportUpload(w http.ResponseWriter, r *http.Request, domainUUID string) {
|
|
if s.importRoot == "" {
|
|
http.Error(w, "import storage unavailable", 500)
|
|
return
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, maxImportArchiveBytes+(1<<20))
|
|
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
|
http.Error(w, err.Error(), 413)
|
|
return
|
|
}
|
|
file, header, err := r.FormFile("archive")
|
|
if err != nil {
|
|
http.Error(w, "archive is required", 400)
|
|
return
|
|
}
|
|
defer file.Close()
|
|
name := filepath.Base(header.Filename)
|
|
if !supportedImportArchive(name) {
|
|
http.Error(w, "supported archives: .zip, .tar, .tar.gz, .tgz, .tar.xz, .txz", 400)
|
|
return
|
|
}
|
|
target := normalizeImportTarget(r.FormValue("target_prefix"))
|
|
actor := ""
|
|
mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode")
|
|
if mode == "enforcing" {
|
|
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
|
|
if authErr != nil || s.store.Authorize(store.AuthorizationRequest{DomainUUID: domainUUID, PrincipalUUID: principal.UUID, Capability: "upload", TargetType: "path", TargetID: target}) != nil {
|
|
http.Error(w, "forbidden", 403)
|
|
return
|
|
}
|
|
actor = principal.UUID
|
|
}
|
|
temp, err := os.CreateTemp(s.importRoot, "upload-*")
|
|
if err != nil {
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
tempPath := temp.Name()
|
|
defer func() { _ = temp.Close() }()
|
|
n, err := io.Copy(temp, io.LimitReader(file, maxImportArchiveBytes+1))
|
|
if err != nil || n > maxImportArchiveBytes {
|
|
_ = os.Remove(tempPath)
|
|
http.Error(w, "archive is too large or unreadable", 413)
|
|
return
|
|
}
|
|
_ = temp.Close()
|
|
id, err := s.store.CreateImportSession(domainUUID, actor, name, tempPath, target)
|
|
if err != nil {
|
|
_ = os.Remove(tempPath)
|
|
http.Error(w, err.Error(), 500)
|
|
return
|
|
}
|
|
payload, _ := json.Marshal(importAnalyzePayload{DomainUUID: domainUUID, SessionUUID: id})
|
|
queued, err := s.store.EnqueueEvent(importAnalyzeEventType, id, string(payload), time.Now())
|
|
if err != nil || !queued {
|
|
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
|
|
http.Error(w, "could not queue import", 500)
|
|
return
|
|
}
|
|
s.wakeEventScheduler()
|
|
http.Redirect(w, r, "/v/import/"+id, 303)
|
|
}
|
|
|
|
func supportedImportArchive(name string) bool {
|
|
name = strings.ToLower(name)
|
|
for _, suffix := range []string{".zip", ".tar", ".tar.gz", ".tgz", ".tar.xz", ".txz"} {
|
|
if strings.HasSuffix(name, suffix) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func normalizeImportTarget(value string) string {
|
|
value = "/" + strings.Trim(strings.TrimSpace(value), "/")
|
|
if value == "/" {
|
|
return value
|
|
}
|
|
return path.Clean(value)
|
|
}
|
|
func safeArchiveName(name string) (string, error) {
|
|
name = strings.ReplaceAll(name, "\\", "/")
|
|
if strings.HasPrefix(name, "/") || strings.ContainsRune(name, '\x00') {
|
|
return "", errors.New("unsafe archive path")
|
|
}
|
|
for _, part := range strings.Split(name, "/") {
|
|
if part == ".." {
|
|
return "", errors.New("unsafe archive path")
|
|
}
|
|
}
|
|
clean := path.Clean(name)
|
|
if clean == "" || clean == "." || strings.HasPrefix(clean, "../") {
|
|
return "", errors.New("unsafe archive path")
|
|
}
|
|
return clean, nil
|
|
}
|
|
|
|
func (s *Server) analyzeImport(payload importAnalyzePayload) error {
|
|
s.store.MarkImportAnalyzing(payload.SessionUUID)
|
|
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
items, err := scanImportArchive(session.ArchivePath, session.ArchiveName, session.TargetPrefix)
|
|
if err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
s.mergeExistingCitySuggestions(session.DomainUUID, session.TargetPrefix, items)
|
|
if err := s.store.ReplaceImportItems(session.UUID, items); err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) mergeExistingCitySuggestions(domainUUID, targetPrefix string, items []store.ImportItem) {
|
|
for i := range items {
|
|
source := strings.Split(strings.Trim(items[i].SourcePath, "/"), "/")
|
|
if len(source) < 2 {
|
|
continue
|
|
}
|
|
qPath := path.Join(targetPrefix, source[0])
|
|
q, err := s.store.CityNodeOverlappingPath(domainUUID, "quartier", "", qPath)
|
|
if err != nil {
|
|
q, err = s.store.CityNode(domainUUID, "quartier", items[i].Quartier)
|
|
}
|
|
if err != nil {
|
|
continue
|
|
}
|
|
items[i].Quartier = q.Name
|
|
streetPath := path.Join(qPath, source[1])
|
|
street, err := s.store.CityNodeOverlappingPath(domainUUID, "street", q.UUID, streetPath)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
items[i].Street = street.Name
|
|
spotName := strings.TrimSpace(items[i].Spot)
|
|
spotPath := path.Join(targetPrefix, path.Dir(items[i].SourcePath))
|
|
if spotName == "" || spotName == street.Name {
|
|
spotPath = path.Join(streetPath, "_undefined")
|
|
} else if len(source) <= 3 {
|
|
spotPath = path.Join(streetPath, "_"+spotName)
|
|
}
|
|
spot, err := s.store.CityNodeOverlappingPath(domainUUID, "spot", street.UUID, spotPath)
|
|
if err == nil && spot.Name != street.Name+"-undefined" {
|
|
items[i].Spot = spot.Name
|
|
}
|
|
}
|
|
}
|
|
|
|
func scanImportArchive(filename, archiveName, target string) ([]store.ImportItem, error) {
|
|
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
|
|
return scanImportZIP(filename, target)
|
|
}
|
|
return scanImportTar(filename, archiveName, target)
|
|
}
|
|
func appendImportItem(items *[]store.ImportItem, name string, size int64, target string) error {
|
|
clean, err := safeArchiveName(name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if strings.HasSuffix(clean, "/") {
|
|
return nil
|
|
}
|
|
if len(*items) >= maxImportFiles {
|
|
return errors.New("archive contains too many files")
|
|
}
|
|
parts := strings.Split(clean, "/")
|
|
title := ""
|
|
media := mime.TypeByExtension(strings.ToLower(filepath.Ext(clean)))
|
|
if strings.EqualFold(filepath.Ext(clean), ".md") {
|
|
media = "text/markdown"
|
|
title = strings.TrimSuffix(parts[len(parts)-1], filepath.Ext(clean))
|
|
}
|
|
if media == "" {
|
|
media = "application/octet-stream"
|
|
}
|
|
labels := parts
|
|
if len(labels) > 0 {
|
|
labels = labels[:len(labels)-1]
|
|
}
|
|
q, street, spot := "", "", ""
|
|
if len(labels) > 0 {
|
|
q = citySuggestionName(labels[0])
|
|
}
|
|
if len(labels) > 1 {
|
|
street = citySuggestionName(labels[1])
|
|
}
|
|
if len(labels) > 2 {
|
|
spot = citySuggestionName(labels[len(labels)-1])
|
|
}
|
|
confidence := 0.45
|
|
if media == "text/markdown" {
|
|
confidence = .7
|
|
}
|
|
targetPath := path.Join(target, clean)
|
|
if media == "text/markdown" {
|
|
targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath))
|
|
}
|
|
*items = append(*items, store.ImportItem{Sequence: len(*items) + 1, SourcePath: clean, TargetPath: targetPath, Size: size, ContentType: media, Title: title, Quartier: q, Street: street, Spot: spot, Confidence: confidence, Selected: true})
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) commitImport(payload importCommitPayload) error {
|
|
s.store.MarkImportStatus(payload.SessionUUID, "importing")
|
|
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
items, err := s.store.ListImportItems(payload.SessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
wanted := map[string]bool{}
|
|
for _, x := range items {
|
|
if x.Selected && !x.Imported {
|
|
wanted[x.SourcePath] = true
|
|
}
|
|
}
|
|
journalEntries, err := s.store.ImportJournal(payload.SessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
journal := map[int]store.ImportJournalEntry{}
|
|
for _, entry := range journalEntries {
|
|
journal[entry.Sequence] = entry
|
|
}
|
|
contents, err := readImportArchive(session.ArchivePath, session.ArchiveName, wanted)
|
|
if err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
cityState := &importCityState{quartiers: map[string]string{}, streets: map[string]string{}, spots: map[string]string{}}
|
|
for _, item := range items {
|
|
if !item.Selected || item.Imported {
|
|
continue
|
|
}
|
|
body, ok := contents[item.SourcePath]
|
|
if !ok {
|
|
err = fmt.Errorf("archive entry disappeared: %s", item.SourcePath)
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
target := item.TargetPath
|
|
if item.ContentType == "text/markdown" {
|
|
target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown")
|
|
}
|
|
prior, hasPrior := journal[item.Sequence]
|
|
if hasPrior {
|
|
target = prior.Path
|
|
}
|
|
if !hasPrior {
|
|
if _, resolveErr := s.store.ResolvePathInDomain(payload.DomainUUID, target); resolveErr == nil {
|
|
if payload.Conflict == "skip" {
|
|
continue
|
|
}
|
|
target = nextImportPath(s.store, payload.DomainUUID, target)
|
|
}
|
|
}
|
|
objectUUID := ""
|
|
var createErr error
|
|
if hasPrior {
|
|
objectUUID = prior.ObjectUUID
|
|
} else {
|
|
objectUUID, createErr = s.store.CreatePageAtPathInDomain(payload.DomainUUID, target)
|
|
}
|
|
if createErr != nil {
|
|
s.store.FailImportSession(session.UUID, createErr)
|
|
return createErr
|
|
}
|
|
contentType := item.ContentType
|
|
filename := filepath.Base(item.SourcePath)
|
|
if item.ContentType == "text/markdown" {
|
|
body = []byte(render.Markdown(string(body)))
|
|
contentType = "text/html; charset=utf-8"
|
|
}
|
|
if !hasPrior {
|
|
if err = s.store.AddImportJournal(session.UUID, item.Sequence, target, objectUUID); err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
}
|
|
if _, err = s.store.SaveRevisionContent(objectUUID, body, "archive import: "+session.ArchiveName, s.languageForDomainUUID(payload.DomainUUID), contentType, filename); err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
if err = s.applyImportCityItem(session, item, target, objectUUID, cityState); err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
if err = s.store.CompleteImportJournal(session.UUID, item.Sequence); err != nil {
|
|
s.store.FailImportSession(session.UUID, err)
|
|
return err
|
|
}
|
|
}
|
|
s.store.MarkImportStatus(session.UUID, "completed")
|
|
return nil
|
|
}
|
|
|
|
type importCityState struct{ quartiers, streets, spots map[string]string }
|
|
|
|
func (s *Server) applyImportCityItem(session store.ImportSession, item store.ImportItem, target, pageUUID string, state *importCityState) error {
|
|
qName, streetName := strings.TrimSpace(item.Quartier), strings.TrimSpace(item.Street)
|
|
if qName == "" || streetName == "" {
|
|
return nil
|
|
}
|
|
actor := session.ActorUUID
|
|
if actor == "" {
|
|
actor = "system:import"
|
|
}
|
|
source := strings.Split(strings.Trim(item.SourcePath, "/"), "/")
|
|
if len(source) < 2 {
|
|
return nil
|
|
}
|
|
qPath := path.Join(session.TargetPrefix, source[0])
|
|
streetPath := path.Join(qPath, source[1])
|
|
qID := state.quartiers[qName]
|
|
if qID == "" {
|
|
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "quartier", "", qPath)
|
|
if err != nil {
|
|
node, err = s.store.CityNode(session.DomainUUID, "quartier", qName)
|
|
}
|
|
if err == nil {
|
|
qID = node.UUID
|
|
if node.PathPrefix == "" || node.PathPrefix == "/" {
|
|
_ = s.store.UpdateCityNodePath(session.DomainUUID, actor, "quartier", qID, qPath)
|
|
}
|
|
} else {
|
|
qID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "quartier", "", qName, qName, "", qPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = s.store.AddImportCityJournal(session.UUID, "quartier", qID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
state.quartiers[qName] = qID
|
|
}
|
|
streetKey := qID + "/" + streetName
|
|
streetID := state.streets[streetKey]
|
|
if streetID == "" {
|
|
overlap, overlapErr := s.store.CityNodeOverlappingPath(session.DomainUUID, "street", qID, streetPath)
|
|
if overlapErr == nil {
|
|
streetID = overlap.UUID
|
|
}
|
|
city, err := s.store.GetCityMap(session.DomainUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if streetID == "" {
|
|
for _, q := range city.Quartiers {
|
|
if q.UUID == qID {
|
|
for _, st := range q.Streets {
|
|
if st.Name == streetName {
|
|
streetID = st.UUID
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if streetID == "" {
|
|
streetID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "street", qID, streetName, streetName, "", streetPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = s.store.AddImportCityJournal(session.UUID, "street", streetID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
state.streets[streetKey] = streetID
|
|
}
|
|
spotName := strings.TrimSpace(item.Spot)
|
|
spotTitle := spotName
|
|
spotPath := path.Join(session.TargetPrefix, path.Dir(item.SourcePath))
|
|
if spotName == "" || spotName == streetName {
|
|
spotName = streetName + "-undefined"
|
|
spotTitle = "Undefined"
|
|
spotPath = path.Join(streetPath, "_undefined")
|
|
} else if len(source) <= 3 {
|
|
// A named Spot for a Page immediately below its Street has no natural
|
|
// directory boundary. Give it an explicit synthetic child boundary so
|
|
// it remains a strict descendant while the Page is linked explicitly.
|
|
spotPath = path.Join(streetPath, "_"+spotName)
|
|
}
|
|
spotKey := streetID + "/" + spotName
|
|
spotID := state.spots[spotKey]
|
|
if spotID == "" {
|
|
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "spot", streetID, spotPath)
|
|
if err != nil {
|
|
node, err = s.store.CityNode(session.DomainUUID, "spot", spotName)
|
|
}
|
|
if err == nil {
|
|
spotID = node.UUID
|
|
} else {
|
|
spotID, err = s.store.CreateSpotAtPath(session.DomainUUID, actor, streetID, spotName, spotTitle, "Imported without a defined Spot", spotPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = s.store.AddImportCityJournal(session.UUID, "spot", spotID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
state.spots[spotKey] = spotID
|
|
}
|
|
if err := s.store.AddSpotPage(session.DomainUUID, actor, spotID, pageUUID, target, "primary", item.Title); err != nil {
|
|
return err
|
|
}
|
|
return s.store.AddImportCityEdge(session.UUID, spotID, pageUUID)
|
|
}
|
|
|
|
func nextImportPath(st *store.Store, domainUUID, target string) string {
|
|
ext := path.Ext(target)
|
|
base := strings.TrimSuffix(target, ext)
|
|
for i := 2; ; i++ {
|
|
candidate := base + "-" + strconv.Itoa(i) + ext
|
|
if _, err := st.ResolvePathInDomain(domainUUID, candidate); err != nil {
|
|
return candidate
|
|
}
|
|
}
|
|
}
|
|
func (s *Server) languageForDomainUUID(domainUUID string) string {
|
|
domain, err := s.store.GetDomainByUUID(domainUUID)
|
|
if err != nil {
|
|
return "und"
|
|
}
|
|
campus := s.campusForDomain(domain)
|
|
if campus == nil || strings.TrimSpace(campus.DefaultLang) == "" {
|
|
return "und"
|
|
}
|
|
return campus.DefaultLang
|
|
}
|
|
|
|
func (s *Server) rollbackImport(domainUUID, sessionUUID string) error {
|
|
session, err := s.store.ImportSession(domainUUID, sessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if session.Status != "completed" && session.Status != "failed" {
|
|
return errors.New("only a completed or failed import can be rolled back")
|
|
}
|
|
actor := session.ActorUUID
|
|
if actor == "" {
|
|
actor = "system:import"
|
|
}
|
|
edges, err := s.store.ImportCityEdges(sessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, edge := range edges {
|
|
if err := s.store.RemoveSpotPage(domainUUID, actor, edge.SpotUUID, edge.PageUUID, "primary"); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
return err
|
|
}
|
|
}
|
|
nodes, err := s.store.ImportCityJournal(sessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, node := range nodes {
|
|
if err := s.store.DeleteCityNode(domainUUID, actor, node.Kind, node.UUID); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
return err
|
|
}
|
|
}
|
|
return s.store.RollbackImportPaths(domainUUID, sessionUUID)
|
|
}
|
|
|
|
func readImportArchive(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
|
|
if strings.HasSuffix(strings.ToLower(name), ".zip") {
|
|
r, err := zip.OpenReader(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer r.Close()
|
|
out := map[string][]byte{}
|
|
for _, f := range r.File {
|
|
clean, err := safeArchiveName(f.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !wanted[clean] {
|
|
continue
|
|
}
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(rc, maxImportExpandedBytes+1))
|
|
rc.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out[clean] = data
|
|
}
|
|
return out, nil
|
|
}
|
|
return readImportTar(filename, name, wanted)
|
|
}
|
|
func readImportTar(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
|
|
file, err := os.Open(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var reader io.Reader = file
|
|
closers := []io.Closer{file}
|
|
lower := strings.ToLower(name)
|
|
var cmd *exec.Cmd
|
|
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
|
|
gz, e := gzip.NewReader(file)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
reader = gz
|
|
closers = append([]io.Closer{gz}, closers...)
|
|
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
|
|
file.Close()
|
|
cmd = exec.Command("xz", "-dc", filename)
|
|
pipe, e := cmd.StdoutPipe()
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if e = cmd.Start(); e != nil {
|
|
return nil, e
|
|
}
|
|
reader = pipe
|
|
closers = []io.Closer{pipe}
|
|
}
|
|
defer func() {
|
|
for _, c := range closers {
|
|
c.Close()
|
|
}
|
|
if cmd != nil {
|
|
cmd.Wait()
|
|
}
|
|
}()
|
|
out := map[string][]byte{}
|
|
tr := tar.NewReader(reader)
|
|
for {
|
|
h, e := tr.Next()
|
|
if e == io.EOF {
|
|
break
|
|
}
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
clean, e := safeArchiveName(h.Name)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if h.Typeflag == tar.TypeReg && wanted[clean] {
|
|
data, e := io.ReadAll(io.LimitReader(tr, maxImportExpandedBytes+1))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
out[clean] = data
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
var suggestionNameRE = regexp.MustCompile(`[^a-z0-9_-]+`)
|
|
|
|
func citySuggestionName(v string) string {
|
|
v = strings.ToLower(v)
|
|
v = suggestionNameRE.ReplaceAllString(v, "-")
|
|
return strings.Trim(v, "-")
|
|
}
|
|
|
|
func scanImportZIP(filename, target string) ([]store.ImportItem, error) {
|
|
r, err := zip.OpenReader(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer r.Close()
|
|
var items []store.ImportItem
|
|
var total int64
|
|
for _, f := range r.File {
|
|
if f.FileInfo().Mode()&os.ModeSymlink != 0 {
|
|
return nil, errors.New("symbolic links are not allowed")
|
|
}
|
|
total += int64(f.UncompressedSize64)
|
|
if total > maxImportExpandedBytes {
|
|
return nil, errors.New("expanded archive is too large")
|
|
}
|
|
if !f.FileInfo().IsDir() {
|
|
if err := appendImportItem(&items, f.Name, int64(f.UncompressedSize64), target); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func scanImportTar(filename, archiveName, target string) ([]store.ImportItem, error) {
|
|
file, err := os.Open(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var reader io.Reader = file
|
|
closers := []io.Closer{file}
|
|
lower := strings.ToLower(archiveName)
|
|
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
|
|
gz, e := gzip.NewReader(file)
|
|
if e != nil {
|
|
file.Close()
|
|
return nil, e
|
|
}
|
|
reader = gz
|
|
closers = append([]io.Closer{gz}, closers...)
|
|
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
|
|
_ = file.Close()
|
|
cmd := exec.Command("xz", "-dc", filename)
|
|
pipe, e := cmd.StdoutPipe()
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if e = cmd.Start(); e != nil {
|
|
return nil, e
|
|
}
|
|
reader = pipe
|
|
closers = []io.Closer{pipe}
|
|
defer cmd.Wait()
|
|
}
|
|
defer func() {
|
|
for _, c := range closers {
|
|
_ = c.Close()
|
|
}
|
|
}()
|
|
tr := tar.NewReader(reader)
|
|
var items []store.ImportItem
|
|
var total int64
|
|
for {
|
|
h, e := tr.Next()
|
|
if e == io.EOF {
|
|
break
|
|
}
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if h.Typeflag == tar.TypeSymlink || h.Typeflag == tar.TypeLink {
|
|
return nil, errors.New("archive links are not allowed")
|
|
}
|
|
if h.Typeflag != tar.TypeReg {
|
|
continue
|
|
}
|
|
total += h.Size
|
|
if total > maxImportExpandedBytes {
|
|
return nil, errors.New("expanded archive is too large")
|
|
}
|
|
if err := appendImportItem(&items, h.Name, h.Size, target); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return items, nil
|
|
}
|