Implement Knowledge City platform, organizer and DAV integration
Capture accumulated application, UI, CLI, regression tests and design documentation changes, including task project mapping and default CalDAV list naming. Keep deployment data, backups, local credentials and build artifacts out of Git. Move the Compose enrollment token to an ignored local environment file.
This commit is contained in:
1 parent
288e6e94ac
commit
c1175d389f
699 files changed
+110197
-4117
No files matched your search
@@ -0,0 +1,828 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/render"
|
||||
"pageswiki/internal/store"
|
||||
)
|
||||
|
||||
const importAnalyzeEventType = "import.analyze"
|
||||
const importCommitEventType = "import.commit"
|
||||
const maxImportArchiveBytes = int64(512 << 20)
|
||||
const maxImportFiles = 10000
|
||||
const maxImportExpandedBytes = int64(2 << 30)
|
||||
|
||||
type importAnalyzePayload struct{ DomainUUID, SessionUUID string }
|
||||
type importCommitPayload struct{ DomainUUID, SessionUUID, Conflict string }
|
||||
|
||||
func (s *Server) initializeImportRoot() {
|
||||
base := filepath.Dir(s.store.DocumentRoot)
|
||||
if s.store.DocumentRoot == "" {
|
||||
base = "data"
|
||||
}
|
||||
s.importRoot = filepath.Join(base, "imports")
|
||||
if err := os.MkdirAll(s.importRoot, 0700); err != nil {
|
||||
s.importRoot = ""
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleImport(w http.ResponseWriter, r *http.Request) {
|
||||
domainUUID := s.domainUUIDForRequest(r)
|
||||
if r.Method == http.MethodPost {
|
||||
s.handleImportUpload(w, r, domainUUID)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/v/import"), "/")
|
||||
data := map[string]any{"Title": translate(s.getLocale(domainUUID), "import.title"), "Locale": s.getLocale(domainUUID)}
|
||||
if id == "" {
|
||||
sessions, err := s.store.ListImportSessions(domainUUID)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
data["Sessions"] = sessions
|
||||
} else {
|
||||
session, err := s.store.ImportSession(domainUUID, id)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
items, err := s.store.ListImportItems(id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
s.mergeExistingCitySuggestions(domainUUID, session.TargetPrefix, items)
|
||||
if session.Status == "failed" {
|
||||
pending := items[:0]
|
||||
for _, item := range items {
|
||||
if !item.Imported {
|
||||
pending = append(pending, item)
|
||||
}
|
||||
}
|
||||
items = pending
|
||||
}
|
||||
data["Session"], data["Items"] = session, items
|
||||
}
|
||||
s.renderWithContext(w, r, "import.gohtml", data)
|
||||
}
|
||||
|
||||
func (s *Server) handleImportAPI(w http.ResponseWriter, r *http.Request) {
|
||||
domainUUID := s.domainUUIDForRequest(r)
|
||||
id := strings.Trim(strings.TrimPrefix(r.URL.Path, "/_/api/v1/import/"), "/")
|
||||
if id == "" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
session, err := s.store.ImportSession(domainUUID, id)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode"); mode == "enforcing" {
|
||||
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
|
||||
if authErr != nil || (session.ActorUUID != principal.UUID && !s.store.IsBuiltinGroupAdmin(domainUUID, principal.UUID, "domain_admin")) {
|
||||
writeEventJSON(w, http.StatusForbidden, map[string]string{"error": "import owner or domain admin required"})
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.Method == http.MethodPatch {
|
||||
var request struct {
|
||||
Items []store.ImportItem `json:"items"`
|
||||
}
|
||||
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4<<20)).Decode(&request) != nil {
|
||||
writeEventJSON(w, 400, map[string]string{"error": "invalid JSON"})
|
||||
return
|
||||
}
|
||||
if err := s.store.UpdateImportPlan(domainUUID, id, request.Items); err != nil {
|
||||
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]string{"status": "ok"})
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
var request struct {
|
||||
Conflict string `json:"conflict"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&request)
|
||||
if request.Conflict != "rename" {
|
||||
request.Conflict = "skip"
|
||||
}
|
||||
if err := s.store.QueueImportCommit(domainUUID, id); err != nil {
|
||||
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
payload, _ := json.Marshal(importCommitPayload{DomainUUID: domainUUID, SessionUUID: id, Conflict: request.Conflict})
|
||||
queued, err := s.store.EnqueueEvent(importCommitEventType, id, string(payload), time.Now())
|
||||
if err != nil || !queued {
|
||||
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
|
||||
writeEventJSON(w, 500, map[string]string{"error": "could not queue import"})
|
||||
return
|
||||
}
|
||||
s.wakeEventScheduler()
|
||||
writeEventJSON(w, 202, map[string]string{"status": "queued"})
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodDelete {
|
||||
if err := s.rollbackImport(domainUUID, id); err != nil {
|
||||
writeEventJSON(w, 400, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeEventJSON(w, 200, map[string]string{"status": "rolled_back"})
|
||||
return
|
||||
}
|
||||
w.WriteHeader(405)
|
||||
}
|
||||
|
||||
func (s *Server) handleImportUpload(w http.ResponseWriter, r *http.Request, domainUUID string) {
|
||||
if s.importRoot == "" {
|
||||
http.Error(w, "import storage unavailable", 500)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxImportArchiveBytes+(1<<20))
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
http.Error(w, err.Error(), 413)
|
||||
return
|
||||
}
|
||||
file, header, err := r.FormFile("archive")
|
||||
if err != nil {
|
||||
http.Error(w, "archive is required", 400)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
name := filepath.Base(header.Filename)
|
||||
if !supportedImportArchive(name) {
|
||||
http.Error(w, "supported archives: .zip, .tar, .tar.gz, .tgz, .tar.xz, .txz", 400)
|
||||
return
|
||||
}
|
||||
target := normalizeImportTarget(r.FormValue("target_prefix"))
|
||||
actor := ""
|
||||
mode, _ := s.store.GetConfigInDomain(domainUUID, "authorization_mode")
|
||||
if mode == "enforcing" {
|
||||
principal, authErr := s.store.PrincipalForSession(domainUUID, s.authTokenFromRequest(r))
|
||||
if authErr != nil || s.store.Authorize(store.AuthorizationRequest{DomainUUID: domainUUID, PrincipalUUID: principal.UUID, Capability: "upload", TargetType: "path", TargetID: target}) != nil {
|
||||
http.Error(w, "forbidden", 403)
|
||||
return
|
||||
}
|
||||
actor = principal.UUID
|
||||
}
|
||||
temp, err := os.CreateTemp(s.importRoot, "upload-*")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
tempPath := temp.Name()
|
||||
defer func() { _ = temp.Close() }()
|
||||
n, err := io.Copy(temp, io.LimitReader(file, maxImportArchiveBytes+1))
|
||||
if err != nil || n > maxImportArchiveBytes {
|
||||
_ = os.Remove(tempPath)
|
||||
http.Error(w, "archive is too large or unreadable", 413)
|
||||
return
|
||||
}
|
||||
_ = temp.Close()
|
||||
id, err := s.store.CreateImportSession(domainUUID, actor, name, tempPath, target)
|
||||
if err != nil {
|
||||
_ = os.Remove(tempPath)
|
||||
http.Error(w, err.Error(), 500)
|
||||
return
|
||||
}
|
||||
payload, _ := json.Marshal(importAnalyzePayload{DomainUUID: domainUUID, SessionUUID: id})
|
||||
queued, err := s.store.EnqueueEvent(importAnalyzeEventType, id, string(payload), time.Now())
|
||||
if err != nil || !queued {
|
||||
s.store.FailImportSession(id, fmt.Errorf("queue: %v", err))
|
||||
http.Error(w, "could not queue import", 500)
|
||||
return
|
||||
}
|
||||
s.wakeEventScheduler()
|
||||
http.Redirect(w, r, "/v/import/"+id, 303)
|
||||
}
|
||||
|
||||
func supportedImportArchive(name string) bool {
|
||||
name = strings.ToLower(name)
|
||||
for _, suffix := range []string{".zip", ".tar", ".tar.gz", ".tgz", ".tar.xz", ".txz"} {
|
||||
if strings.HasSuffix(name, suffix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func normalizeImportTarget(value string) string {
|
||||
value = "/" + strings.Trim(strings.TrimSpace(value), "/")
|
||||
if value == "/" {
|
||||
return value
|
||||
}
|
||||
return path.Clean(value)
|
||||
}
|
||||
func safeArchiveName(name string) (string, error) {
|
||||
name = strings.ReplaceAll(name, "\\", "/")
|
||||
if strings.HasPrefix(name, "/") || strings.ContainsRune(name, '\x00') {
|
||||
return "", errors.New("unsafe archive path")
|
||||
}
|
||||
for _, part := range strings.Split(name, "/") {
|
||||
if part == ".." {
|
||||
return "", errors.New("unsafe archive path")
|
||||
}
|
||||
}
|
||||
clean := path.Clean(name)
|
||||
if clean == "" || clean == "." || strings.HasPrefix(clean, "../") {
|
||||
return "", errors.New("unsafe archive path")
|
||||
}
|
||||
return clean, nil
|
||||
}
|
||||
|
||||
func (s *Server) analyzeImport(payload importAnalyzePayload) error {
|
||||
s.store.MarkImportAnalyzing(payload.SessionUUID)
|
||||
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items, err := scanImportArchive(session.ArchivePath, session.ArchiveName, session.TargetPrefix)
|
||||
if err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
s.mergeExistingCitySuggestions(session.DomainUUID, session.TargetPrefix, items)
|
||||
if err := s.store.ReplaceImportItems(session.UUID, items); err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) mergeExistingCitySuggestions(domainUUID, targetPrefix string, items []store.ImportItem) {
|
||||
for i := range items {
|
||||
source := strings.Split(strings.Trim(items[i].SourcePath, "/"), "/")
|
||||
if len(source) < 2 {
|
||||
continue
|
||||
}
|
||||
qPath := path.Join(targetPrefix, source[0])
|
||||
q, err := s.store.CityNodeOverlappingPath(domainUUID, "quartier", "", qPath)
|
||||
if err != nil {
|
||||
q, err = s.store.CityNode(domainUUID, "quartier", items[i].Quartier)
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
items[i].Quartier = q.Name
|
||||
streetPath := path.Join(qPath, source[1])
|
||||
street, err := s.store.CityNodeOverlappingPath(domainUUID, "street", q.UUID, streetPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
items[i].Street = street.Name
|
||||
spotName := strings.TrimSpace(items[i].Spot)
|
||||
spotPath := path.Join(targetPrefix, path.Dir(items[i].SourcePath))
|
||||
if spotName == "" || spotName == street.Name {
|
||||
spotPath = path.Join(streetPath, "_undefined")
|
||||
} else if len(source) <= 3 {
|
||||
spotPath = path.Join(streetPath, "_"+spotName)
|
||||
}
|
||||
spot, err := s.store.CityNodeOverlappingPath(domainUUID, "spot", street.UUID, spotPath)
|
||||
if err == nil && spot.Name != street.Name+"-undefined" {
|
||||
items[i].Spot = spot.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func scanImportArchive(filename, archiveName, target string) ([]store.ImportItem, error) {
|
||||
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
|
||||
return scanImportZIP(filename, target)
|
||||
}
|
||||
return scanImportTar(filename, archiveName, target)
|
||||
}
|
||||
func appendImportItem(items *[]store.ImportItem, name string, size int64, target string) error {
|
||||
clean, err := safeArchiveName(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasSuffix(clean, "/") {
|
||||
return nil
|
||||
}
|
||||
if len(*items) >= maxImportFiles {
|
||||
return errors.New("archive contains too many files")
|
||||
}
|
||||
parts := strings.Split(clean, "/")
|
||||
title := ""
|
||||
media := mime.TypeByExtension(strings.ToLower(filepath.Ext(clean)))
|
||||
if strings.EqualFold(filepath.Ext(clean), ".md") {
|
||||
media = "text/markdown"
|
||||
title = strings.TrimSuffix(parts[len(parts)-1], filepath.Ext(clean))
|
||||
}
|
||||
if media == "" {
|
||||
media = "application/octet-stream"
|
||||
}
|
||||
labels := parts
|
||||
if len(labels) > 0 {
|
||||
labels = labels[:len(labels)-1]
|
||||
}
|
||||
q, street, spot := "", "", ""
|
||||
if len(labels) > 0 {
|
||||
q = citySuggestionName(labels[0])
|
||||
}
|
||||
if len(labels) > 1 {
|
||||
street = citySuggestionName(labels[1])
|
||||
}
|
||||
if len(labels) > 2 {
|
||||
spot = citySuggestionName(labels[len(labels)-1])
|
||||
}
|
||||
confidence := 0.45
|
||||
if media == "text/markdown" {
|
||||
confidence = .7
|
||||
}
|
||||
targetPath := path.Join(target, clean)
|
||||
if media == "text/markdown" {
|
||||
targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath))
|
||||
}
|
||||
*items = append(*items, store.ImportItem{Sequence: len(*items) + 1, SourcePath: clean, TargetPath: targetPath, Size: size, ContentType: media, Title: title, Quartier: q, Street: street, Spot: spot, Confidence: confidence, Selected: true})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) commitImport(payload importCommitPayload) error {
|
||||
s.store.MarkImportStatus(payload.SessionUUID, "importing")
|
||||
session, err := s.store.ImportSession(payload.DomainUUID, payload.SessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items, err := s.store.ListImportItems(payload.SessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wanted := map[string]bool{}
|
||||
for _, x := range items {
|
||||
if x.Selected && !x.Imported {
|
||||
wanted[x.SourcePath] = true
|
||||
}
|
||||
}
|
||||
journalEntries, err := s.store.ImportJournal(payload.SessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
journal := map[int]store.ImportJournalEntry{}
|
||||
for _, entry := range journalEntries {
|
||||
journal[entry.Sequence] = entry
|
||||
}
|
||||
contents, err := readImportArchive(session.ArchivePath, session.ArchiveName, wanted)
|
||||
if err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
cityState := &importCityState{quartiers: map[string]string{}, streets: map[string]string{}, spots: map[string]string{}}
|
||||
for _, item := range items {
|
||||
if !item.Selected || item.Imported {
|
||||
continue
|
||||
}
|
||||
body, ok := contents[item.SourcePath]
|
||||
if !ok {
|
||||
err = fmt.Errorf("archive entry disappeared: %s", item.SourcePath)
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
target := item.TargetPath
|
||||
if item.ContentType == "text/markdown" {
|
||||
target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown")
|
||||
}
|
||||
prior, hasPrior := journal[item.Sequence]
|
||||
if hasPrior {
|
||||
target = prior.Path
|
||||
}
|
||||
if !hasPrior {
|
||||
if _, resolveErr := s.store.ResolvePathInDomain(payload.DomainUUID, target); resolveErr == nil {
|
||||
if payload.Conflict == "skip" {
|
||||
continue
|
||||
}
|
||||
target = nextImportPath(s.store, payload.DomainUUID, target)
|
||||
}
|
||||
}
|
||||
objectUUID := ""
|
||||
var createErr error
|
||||
if hasPrior {
|
||||
objectUUID = prior.ObjectUUID
|
||||
} else {
|
||||
objectUUID, createErr = s.store.CreatePageAtPathInDomain(payload.DomainUUID, target)
|
||||
}
|
||||
if createErr != nil {
|
||||
s.store.FailImportSession(session.UUID, createErr)
|
||||
return createErr
|
||||
}
|
||||
contentType := item.ContentType
|
||||
filename := filepath.Base(item.SourcePath)
|
||||
if item.ContentType == "text/markdown" {
|
||||
body = []byte(render.Markdown(string(body)))
|
||||
contentType = "text/html; charset=utf-8"
|
||||
}
|
||||
if !hasPrior {
|
||||
if err = s.store.AddImportJournal(session.UUID, item.Sequence, target, objectUUID); err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = s.store.SaveRevisionContent(objectUUID, body, "archive import: "+session.ArchiveName, s.languageForDomainUUID(payload.DomainUUID), contentType, filename); err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
if err = s.applyImportCityItem(session, item, target, objectUUID, cityState); err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
if err = s.store.CompleteImportJournal(session.UUID, item.Sequence); err != nil {
|
||||
s.store.FailImportSession(session.UUID, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.store.MarkImportStatus(session.UUID, "completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
type importCityState struct{ quartiers, streets, spots map[string]string }
|
||||
|
||||
func (s *Server) applyImportCityItem(session store.ImportSession, item store.ImportItem, target, pageUUID string, state *importCityState) error {
|
||||
qName, streetName := strings.TrimSpace(item.Quartier), strings.TrimSpace(item.Street)
|
||||
if qName == "" || streetName == "" {
|
||||
return nil
|
||||
}
|
||||
actor := session.ActorUUID
|
||||
if actor == "" {
|
||||
actor = "system:import"
|
||||
}
|
||||
source := strings.Split(strings.Trim(item.SourcePath, "/"), "/")
|
||||
if len(source) < 2 {
|
||||
return nil
|
||||
}
|
||||
qPath := path.Join(session.TargetPrefix, source[0])
|
||||
streetPath := path.Join(qPath, source[1])
|
||||
qID := state.quartiers[qName]
|
||||
if qID == "" {
|
||||
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "quartier", "", qPath)
|
||||
if err != nil {
|
||||
node, err = s.store.CityNode(session.DomainUUID, "quartier", qName)
|
||||
}
|
||||
if err == nil {
|
||||
qID = node.UUID
|
||||
if node.PathPrefix == "" || node.PathPrefix == "/" {
|
||||
_ = s.store.UpdateCityNodePath(session.DomainUUID, actor, "quartier", qID, qPath)
|
||||
}
|
||||
} else {
|
||||
qID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "quartier", "", qName, qName, "", qPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = s.store.AddImportCityJournal(session.UUID, "quartier", qID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
state.quartiers[qName] = qID
|
||||
}
|
||||
streetKey := qID + "/" + streetName
|
||||
streetID := state.streets[streetKey]
|
||||
if streetID == "" {
|
||||
overlap, overlapErr := s.store.CityNodeOverlappingPath(session.DomainUUID, "street", qID, streetPath)
|
||||
if overlapErr == nil {
|
||||
streetID = overlap.UUID
|
||||
}
|
||||
city, err := s.store.GetCityMap(session.DomainUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if streetID == "" {
|
||||
for _, q := range city.Quartiers {
|
||||
if q.UUID == qID {
|
||||
for _, st := range q.Streets {
|
||||
if st.Name == streetName {
|
||||
streetID = st.UUID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if streetID == "" {
|
||||
streetID, err = s.store.CreateZoneAtPath(session.DomainUUID, actor, "street", qID, streetName, streetName, "", streetPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = s.store.AddImportCityJournal(session.UUID, "street", streetID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
state.streets[streetKey] = streetID
|
||||
}
|
||||
spotName := strings.TrimSpace(item.Spot)
|
||||
spotTitle := spotName
|
||||
spotPath := path.Join(session.TargetPrefix, path.Dir(item.SourcePath))
|
||||
if spotName == "" || spotName == streetName {
|
||||
spotName = streetName + "-undefined"
|
||||
spotTitle = "Undefined"
|
||||
spotPath = path.Join(streetPath, "_undefined")
|
||||
} else if len(source) <= 3 {
|
||||
// A named Spot for a Page immediately below its Street has no natural
|
||||
// directory boundary. Give it an explicit synthetic child boundary so
|
||||
// it remains a strict descendant while the Page is linked explicitly.
|
||||
spotPath = path.Join(streetPath, "_"+spotName)
|
||||
}
|
||||
spotKey := streetID + "/" + spotName
|
||||
spotID := state.spots[spotKey]
|
||||
if spotID == "" {
|
||||
node, err := s.store.CityNodeOverlappingPath(session.DomainUUID, "spot", streetID, spotPath)
|
||||
if err != nil {
|
||||
node, err = s.store.CityNode(session.DomainUUID, "spot", spotName)
|
||||
}
|
||||
if err == nil {
|
||||
spotID = node.UUID
|
||||
} else {
|
||||
spotID, err = s.store.CreateSpotAtPath(session.DomainUUID, actor, streetID, spotName, spotTitle, "Imported without a defined Spot", spotPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = s.store.AddImportCityJournal(session.UUID, "spot", spotID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
state.spots[spotKey] = spotID
|
||||
}
|
||||
if err := s.store.AddSpotPage(session.DomainUUID, actor, spotID, pageUUID, target, "primary", item.Title); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.store.AddImportCityEdge(session.UUID, spotID, pageUUID)
|
||||
}
|
||||
|
||||
func nextImportPath(st *store.Store, domainUUID, target string) string {
|
||||
ext := path.Ext(target)
|
||||
base := strings.TrimSuffix(target, ext)
|
||||
for i := 2; ; i++ {
|
||||
candidate := base + "-" + strconv.Itoa(i) + ext
|
||||
if _, err := st.ResolvePathInDomain(domainUUID, candidate); err != nil {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
func (s *Server) languageForDomainUUID(domainUUID string) string {
|
||||
domain, err := s.store.GetDomainByUUID(domainUUID)
|
||||
if err != nil {
|
||||
return "und"
|
||||
}
|
||||
campus := s.campusForDomain(domain)
|
||||
if campus == nil || strings.TrimSpace(campus.DefaultLang) == "" {
|
||||
return "und"
|
||||
}
|
||||
return campus.DefaultLang
|
||||
}
|
||||
|
||||
func (s *Server) rollbackImport(domainUUID, sessionUUID string) error {
|
||||
session, err := s.store.ImportSession(domainUUID, sessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if session.Status != "completed" && session.Status != "failed" {
|
||||
return errors.New("only a completed or failed import can be rolled back")
|
||||
}
|
||||
entries, err := s.store.ImportJournal(sessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actor := session.ActorUUID
|
||||
if actor == "" {
|
||||
actor = "system:import"
|
||||
}
|
||||
edges, err := s.store.ImportCityEdges(sessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, edge := range edges {
|
||||
_ = s.store.RemoveSpotPage(domainUUID, actor, edge.SpotUUID, edge.PageUUID, "primary")
|
||||
}
|
||||
nodes, err := s.store.ImportCityJournal(sessionUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
_ = s.store.DeleteCityNode(domainUUID, actor, node.Kind, node.UUID)
|
||||
}
|
||||
tx, err := s.store.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, x := range entries {
|
||||
if _, err = tx.Exec(`DELETE FROM paths WHERE domain_uuid=? AND path=? AND object_uuid=?`, domainUUID, x.Path, x.ObjectUUID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(`UPDATE import_sessions SET status='rolled_back',updated_at_ms=? WHERE uuid=?`, time.Now().UnixMilli(), sessionUUID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(); err == nil {
|
||||
s.store.InvalidatePathCaches()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func readImportArchive(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
|
||||
if strings.HasSuffix(strings.ToLower(name), ".zip") {
|
||||
r, err := zip.OpenReader(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
out := map[string][]byte{}
|
||||
for _, f := range r.File {
|
||||
clean, err := safeArchiveName(f.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !wanted[clean] {
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(rc, maxImportExpandedBytes+1))
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[clean] = data
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return readImportTar(filename, name, wanted)
|
||||
}
|
||||
func readImportTar(filename, name string, wanted map[string]bool) (map[string][]byte, error) {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var reader io.Reader = file
|
||||
closers := []io.Closer{file}
|
||||
lower := strings.ToLower(name)
|
||||
var cmd *exec.Cmd
|
||||
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
|
||||
gz, e := gzip.NewReader(file)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
reader = gz
|
||||
closers = append([]io.Closer{gz}, closers...)
|
||||
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
|
||||
file.Close()
|
||||
cmd = exec.Command("xz", "-dc", filename)
|
||||
pipe, e := cmd.StdoutPipe()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e = cmd.Start(); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
reader = pipe
|
||||
closers = []io.Closer{pipe}
|
||||
}
|
||||
defer func() {
|
||||
for _, c := range closers {
|
||||
c.Close()
|
||||
}
|
||||
if cmd != nil {
|
||||
cmd.Wait()
|
||||
}
|
||||
}()
|
||||
out := map[string][]byte{}
|
||||
tr := tar.NewReader(reader)
|
||||
for {
|
||||
h, e := tr.Next()
|
||||
if e == io.EOF {
|
||||
break
|
||||
}
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
clean, e := safeArchiveName(h.Name)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if h.Typeflag == tar.TypeReg && wanted[clean] {
|
||||
data, e := io.ReadAll(io.LimitReader(tr, maxImportExpandedBytes+1))
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
out[clean] = data
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var suggestionNameRE = regexp.MustCompile(`[^a-z0-9_-]+`)
|
||||
|
||||
func citySuggestionName(v string) string {
|
||||
v = strings.ToLower(v)
|
||||
v = suggestionNameRE.ReplaceAllString(v, "-")
|
||||
return strings.Trim(v, "-")
|
||||
}
|
||||
|
||||
func scanImportZIP(filename, target string) ([]store.ImportItem, error) {
|
||||
r, err := zip.OpenReader(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
var items []store.ImportItem
|
||||
var total int64
|
||||
for _, f := range r.File {
|
||||
if f.FileInfo().Mode()&os.ModeSymlink != 0 {
|
||||
return nil, errors.New("symbolic links are not allowed")
|
||||
}
|
||||
total += int64(f.UncompressedSize64)
|
||||
if total > maxImportExpandedBytes {
|
||||
return nil, errors.New("expanded archive is too large")
|
||||
}
|
||||
if !f.FileInfo().IsDir() {
|
||||
if err := appendImportItem(&items, f.Name, int64(f.UncompressedSize64), target); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func scanImportTar(filename, archiveName, target string) ([]store.ImportItem, error) {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var reader io.Reader = file
|
||||
closers := []io.Closer{file}
|
||||
lower := strings.ToLower(archiveName)
|
||||
if strings.HasSuffix(lower, ".gz") || strings.HasSuffix(lower, ".tgz") {
|
||||
gz, e := gzip.NewReader(file)
|
||||
if e != nil {
|
||||
file.Close()
|
||||
return nil, e
|
||||
}
|
||||
reader = gz
|
||||
closers = append([]io.Closer{gz}, closers...)
|
||||
} else if strings.HasSuffix(lower, ".xz") || strings.HasSuffix(lower, ".txz") {
|
||||
_ = file.Close()
|
||||
cmd := exec.Command("xz", "-dc", filename)
|
||||
pipe, e := cmd.StdoutPipe()
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if e = cmd.Start(); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
reader = pipe
|
||||
closers = []io.Closer{pipe}
|
||||
defer cmd.Wait()
|
||||
}
|
||||
defer func() {
|
||||
for _, c := range closers {
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
tr := tar.NewReader(reader)
|
||||
var items []store.ImportItem
|
||||
var total int64
|
||||
for {
|
||||
h, e := tr.Next()
|
||||
if e == io.EOF {
|
||||
break
|
||||
}
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if h.Typeflag == tar.TypeSymlink || h.Typeflag == tar.TypeLink {
|
||||
return nil, errors.New("archive links are not allowed")
|
||||
}
|
||||
if h.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
total += h.Size
|
||||
if total > maxImportExpandedBytes {
|
||||
return nil, errors.New("expanded archive is too large")
|
||||
}
|
||||
if err := appendImportItem(&items, h.Name, h.Size, target); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
Reference in new issue
Block a user