package store import ( "strings" "time" "github.com/google/uuid" ) type ImportSession struct { UUID, DomainUUID, ActorUUID, ArchiveName, ArchivePath, TargetPrefix string Status, Error string TotalFiles, MarkdownFiles int CreatedAtMS, UpdatedAtMS int64 } type ImportItem struct { Sequence int `json:"sequence"` SourcePath string `json:"source_path"` TargetPath string `json:"target_path"` Size int64 `json:"size"` ContentType string `json:"content_type"` Title string `json:"title"` Quartier, Street, Spot string Confidence float64 Selected bool Imported bool } func (s *Store) EnsureImportSchema() error { if s.sqlBackend == storePostgreSQL { return s.ensurePostgreSQLImports() } return s.ensureSchemaComponent("imports") } func (s *Store) CreateImportSession(domainUUID, actor, archiveName, archivePath, targetPrefix string) (string, error) { tx, cancel, err := s.beginPathMutation(domainUUID) defer cancel() if err != nil { return "", err } defer tx.Rollback() id, now := uuid.NewString(), time.Now().UnixMilli() _, err = tx.Exec(`INSERT INTO import_sessions(uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,created_at_ms,updated_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`)`, id, domainUUID, actor, archiveName, archivePath, targetPrefix, now, now) if err == nil { err = tx.Commit() } return id, s.sqlError("create import session", err) } func (s *Store) ImportSession(domainUUID, id string) (ImportSession, error) { var x ImportSession err := s.DB.QueryRow(`SELECT uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms FROM import_sessions WHERE domain_uuid=`+s.sqlParam(1)+` AND uuid=`+s.sqlParam(2)+``, domainUUID, id).Scan(&x.UUID, &x.DomainUUID, &x.ActorUUID, &x.ArchiveName, &x.ArchivePath, &x.TargetPrefix, &x.Status, &x.TotalFiles, &x.MarkdownFiles, &x.Error, &x.CreatedAtMS, &x.UpdatedAtMS) return x, err } func (s *Store) ListImportSessions(domainUUID string) ([]ImportSession, error) { rows, err := s.DB.Query(`SELECT uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms FROM import_sessions WHERE domain_uuid=`+s.sqlParam(1)+` ORDER BY created_at_ms DESC LIMIT 50`, domainUUID) if err != nil { return nil, err } defer rows.Close() var out []ImportSession for rows.Next() { var x ImportSession if err := rows.Scan(&x.UUID, &x.DomainUUID, &x.ActorUUID, &x.ArchiveName, &x.ArchivePath, &x.TargetPrefix, &x.Status, &x.TotalFiles, &x.MarkdownFiles, &x.Error, &x.CreatedAtMS, &x.UpdatedAtMS); err != nil { return nil, err } out = append(out, x) } return out, rows.Err() } func (s *Store) ReplaceImportItems(sessionUUID string, items []ImportItem) (resultErr error) { defer func() { resultErr = s.sqlError("replace import items", resultErr) }() tx, cancel, err := s.beginImportMutation(sessionUUID) defer cancel() if err != nil { return err } defer tx.Rollback() if _, err = tx.Exec(`DELETE FROM import_items WHERE session_uuid=`+s.sqlParam(1)+``, sessionUUID); err != nil { return err } markdown := 0 for _, x := range items { if x.ContentType == "text/markdown" { markdown++ } if _, err = tx.Exec(`INSERT INTO import_items(session_uuid,sequence,source_path,target_path,size,content_type,title,suggested_quartier,suggested_street,suggested_spot,confidence) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`,`+s.sqlParam(9)+`,`+s.sqlParam(10)+`,`+s.sqlParam(11)+`)`, sessionUUID, x.Sequence, x.SourcePath, x.TargetPath, x.Size, x.ContentType, x.Title, x.Quartier, x.Street, x.Spot, x.Confidence); err != nil { return err } } _, err = tx.Exec(`UPDATE import_sessions SET status='ready',total_files=`+s.sqlParam(1)+`,markdown_files=`+s.sqlParam(2)+`,error='',updated_at_ms=`+s.sqlParam(3)+` WHERE uuid=`+s.sqlParam(4)+``, len(items), markdown, time.Now().UnixMilli(), sessionUUID) if err != nil { return err } return tx.Commit() } func (s *Store) FailImportSession(id string, failure error) { _, _ = s.execImportMutation(id, `UPDATE import_sessions SET status='failed',error=`+s.sqlParam(1)+`,updated_at_ms=`+s.sqlParam(2)+` WHERE uuid=`+s.sqlParam(3)+``, failure.Error(), time.Now().UnixMilli(), id) } func (s *Store) MarkImportAnalyzing(id string) { _, _ = s.execImportMutation(id, `UPDATE import_sessions SET status='analyzing',updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+``, time.Now().UnixMilli(), id) } func (s *Store) ListImportItems(sessionUUID string) ([]ImportItem, error) { rows, err := s.DB.Query(`SELECT i.sequence,i.source_path,i.target_path,i.size,i.content_type,i.title,i.suggested_quartier,i.suggested_street,i.suggested_spot,i.confidence,i.selected,COALESCE(j.completed,0) FROM import_items i LEFT JOIN import_journal j ON j.session_uuid=i.session_uuid AND j.sequence=i.sequence WHERE i.session_uuid=`+s.sqlParam(1)+` ORDER BY i.sequence`, sessionUUID) if err != nil { return nil, err } defer rows.Close() var out []ImportItem for rows.Next() { var x ImportItem var selected, imported int if err := rows.Scan(&x.Sequence, &x.SourcePath, &x.TargetPath, &x.Size, &x.ContentType, &x.Title, &x.Quartier, &x.Street, &x.Spot, &x.Confidence, &selected, &imported); err != nil { return nil, err } x.Selected = selected != 0 x.Imported = imported != 0 if x.ContentType == "text/markdown" { x.TargetPath = strings.TrimSuffix(strings.TrimSuffix(x.TargetPath, ".md"), ".markdown") } out = append(out, x) } return out, rows.Err() } func (s *Store) UpdateImportPlan(domainUUID, sessionUUID string, items []ImportItem) (resultErr error) { defer func() { resultErr = s.sqlError("update import plan", resultErr) }() tx, cancel, err := s.beginPathMutation(domainUUID) defer cancel() if err != nil { return err } defer tx.Rollback() var status string if err := tx.QueryRow(`SELECT status FROM import_sessions WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2)+``, sessionUUID, domainUUID).Scan(&status); err != nil { return err } if status != "ready" && status != "failed" { return namespaceValidationError("import plan is not editable") } for _, x := range items { x.Quartier, x.Street, x.Spot = strings.TrimSpace(x.Quartier), strings.TrimSpace(x.Street), strings.TrimSpace(x.Spot) if x.Street != "" && x.Quartier == "" { return namespaceValidationError("Street requires a Quartier") } if x.Spot != "" && x.Street == "" { return namespaceValidationError("Spot requires a Street") } if x.Spot == x.Street { x.Spot = "" } for _, name := range []string{x.Quartier, x.Street, x.Spot} { if name != "" { if _, err := normalizeCityName(name); err != nil { return err } } } if strings.TrimSpace(x.TargetPath) == "" || normalizePath(x.TargetPath) == "/" { return namespaceValidationError("import target Path must identify a document") } var contentType string if err := tx.QueryRow(`SELECT content_type FROM import_items WHERE session_uuid=`+s.sqlParam(1)+` AND sequence=`+s.sqlParam(2)+``, sessionUUID, x.Sequence).Scan(&contentType); err != nil { return err } target := normalizePath(x.TargetPath) if contentType == "text/markdown" { target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown") } result, err := tx.Exec(`UPDATE import_items SET target_path=`+s.sqlParam(1)+`,suggested_quartier=`+s.sqlParam(2)+`,suggested_street=`+s.sqlParam(3)+`,suggested_spot=`+s.sqlParam(4)+`,selected=`+s.sqlParam(5)+` WHERE session_uuid=`+s.sqlParam(6)+` AND sequence=`+s.sqlParam(7)+` AND NOT EXISTS(SELECT 1 FROM import_journal j WHERE j.session_uuid=import_items.session_uuid AND j.sequence=import_items.sequence AND j.completed=1)`, target, x.Quartier, x.Street, x.Spot, boolToInt(x.Selected), sessionUUID, x.Sequence) if err != nil { return err } if n, _ := result.RowsAffected(); n != 1 { continue } } _, err = tx.Exec(`UPDATE import_sessions SET updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+``, time.Now().UnixMilli(), sessionUUID) if err != nil { return err } return tx.Commit() } func (s *Store) QueueImportCommit(domainUUID, sessionUUID string) error { result, err := s.execImportMutation(sessionUUID, `UPDATE import_sessions SET status='queued_import',error='',updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+` AND domain_uuid=`+s.sqlParam(3)+` AND status IN ('ready','failed')`, time.Now().UnixMilli(), sessionUUID, domainUUID) if err != nil { return err } if n, _ := result.RowsAffected(); n != 1 { return namespaceValidationError("import is not ready") } return nil } func (s *Store) MarkImportStatus(id, status string) { _, _ = s.execImportMutation(id, `UPDATE import_sessions SET status=`+s.sqlParam(1)+`,updated_at_ms=`+s.sqlParam(2)+` WHERE uuid=`+s.sqlParam(3)+``, status, time.Now().UnixMilli(), id) } func (s *Store) AddImportJournal(sessionUUID string, sequence int, path, objectUUID string) error { _, err := s.execImportMutation(sessionUUID, `INSERT INTO import_journal(session_uuid,sequence,path,object_uuid,created_at_ms,completed) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,0)`, sessionUUID, sequence, path, objectUUID, time.Now().UnixMilli()) return err } func (s *Store) CompleteImportJournal(sessionUUID string, sequence int) error { _, err := s.execImportMutation(sessionUUID, `UPDATE import_journal SET completed=1 WHERE session_uuid=`+s.sqlParam(1)+` AND sequence=`+s.sqlParam(2)+``, sessionUUID, sequence) return err } func (s *Store) AddImportCityJournal(sessionUUID, kind, nodeUUID string) error { _, err := s.execImportMutation(sessionUUID, `INSERT INTO import_city_journal(session_uuid,kind,node_uuid,created_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`) ON CONFLICT DO NOTHING`, sessionUUID, kind, nodeUUID, time.Now().UnixMilli()) return err } func (s *Store) AddImportCityEdge(sessionUUID, spotUUID, pageUUID string) error { _, err := s.execImportMutation(sessionUUID, `INSERT INTO import_city_edges(session_uuid,spot_uuid,page_uuid,created_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`) ON CONFLICT DO NOTHING`, sessionUUID, spotUUID, pageUUID, time.Now().UnixMilli()) return err } type ImportCityJournalEntry struct{ Kind, UUID string } func (s *Store) ImportCityJournal(sessionUUID string) ([]ImportCityJournalEntry, error) { rows, err := s.DB.Query(`SELECT kind,node_uuid FROM import_city_journal WHERE session_uuid=`+s.sqlParam(1)+` ORDER BY CASE kind WHEN 'spot' THEN 1 WHEN 'street' THEN 2 ELSE 3 END`, sessionUUID) if err != nil { return nil, err } defer rows.Close() var out []ImportCityJournalEntry for rows.Next() { var x ImportCityJournalEntry if err := rows.Scan(&x.Kind, &x.UUID); err != nil { return nil, err } out = append(out, x) } return out, rows.Err() } type ImportCityEdge struct{ SpotUUID, PageUUID string } func (s *Store) ImportCityEdges(sessionUUID string) ([]ImportCityEdge, error) { rows, err := s.DB.Query(`SELECT spot_uuid,page_uuid FROM import_city_edges WHERE session_uuid=`+s.sqlParam(1)+``, sessionUUID) if err != nil { return nil, err } defer rows.Close() var out []ImportCityEdge for rows.Next() { var x ImportCityEdge if err := rows.Scan(&x.SpotUUID, &x.PageUUID); err != nil { return nil, err } out = append(out, x) } return out, rows.Err() } type ImportJournalEntry struct { Sequence int Path, ObjectUUID string Completed bool } func (s *Store) ImportJournal(sessionUUID string) ([]ImportJournalEntry, error) { rows, err := s.DB.Query(`SELECT sequence,path,object_uuid,completed FROM import_journal WHERE session_uuid=`+s.sqlParam(1)+` ORDER BY sequence`, sessionUUID) if err != nil { return nil, err } defer rows.Close() var out []ImportJournalEntry for rows.Next() { var x ImportJournalEntry var completed int if err := rows.Scan(&x.Sequence, &x.Path, &x.ObjectUUID, &completed); err != nil { return nil, err } x.Completed = completed != 0 out = append(out, x) } return out, rows.Err() } // RollbackImportPaths removes only bindings still owned by this import. A Path // subsequently rebound to another Object must survive. City cleanup is a // separate, retryable phase performed by the caller before this transaction. func (s *Store) RollbackImportPaths(domain, session string) (resultErr error) { defer func() { resultErr = s.sqlError("rollback import paths", resultErr) }() tx, cancel, err := s.beginPathMutation(domain) defer cancel() if err != nil { return err } defer tx.Rollback() var status string if err = tx.QueryRow(`SELECT status FROM import_sessions WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2), session, domain).Scan(&status); err != nil { return err } if status != "completed" && status != "failed" { return namespaceValidationError("only a completed or failed import can be rolled back") } if _, err = tx.Exec(`DELETE FROM paths WHERE domain_uuid=`+s.sqlParam(1)+` AND EXISTS(SELECT 1 FROM import_journal j WHERE j.session_uuid=`+s.sqlParam(2)+` AND j.path=paths.path AND j.object_uuid=paths.object_uuid)`, domain, session); err != nil { return err } if _, err = tx.Exec(`UPDATE import_sessions SET status='rolled_back',updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+` AND domain_uuid=`+s.sqlParam(3), time.Now().UnixMilli(), session, domain); err != nil { return err } if err = tx.Commit(); err == nil { s.InvalidatePathCaches() } return err }