313 lines
14 KiB
Go
313 lines
14 KiB
Go
package store
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type ImportSession struct {
|
|
UUID, DomainUUID, ActorUUID, ArchiveName, ArchivePath, TargetPrefix string
|
|
Status, Error string
|
|
TotalFiles, MarkdownFiles int
|
|
CreatedAtMS, UpdatedAtMS int64
|
|
}
|
|
|
|
type ImportItem struct {
|
|
Sequence int `json:"sequence"`
|
|
SourcePath string `json:"source_path"`
|
|
TargetPath string `json:"target_path"`
|
|
Size int64 `json:"size"`
|
|
ContentType string `json:"content_type"`
|
|
Title string `json:"title"`
|
|
Quartier, Street, Spot string
|
|
Confidence float64
|
|
Selected bool
|
|
Imported bool
|
|
}
|
|
|
|
func (s *Store) EnsureImportSchema() error {
|
|
if s.sqlBackend == storePostgreSQL {
|
|
return s.ensurePostgreSQLImports()
|
|
}
|
|
return s.ensureSchemaComponent("imports")
|
|
}
|
|
|
|
func (s *Store) CreateImportSession(domainUUID, actor, archiveName, archivePath, targetPrefix string) (string, error) {
|
|
tx, cancel, err := s.beginPathMutation(domainUUID)
|
|
defer cancel()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer tx.Rollback()
|
|
id, now := uuid.NewString(), time.Now().UnixMilli()
|
|
_, err = tx.Exec(`INSERT INTO import_sessions(uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,created_at_ms,updated_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`)`, id, domainUUID, actor, archiveName, archivePath, targetPrefix, now, now)
|
|
if err == nil {
|
|
err = tx.Commit()
|
|
}
|
|
return id, s.sqlError("create import session", err)
|
|
}
|
|
|
|
func (s *Store) ImportSession(domainUUID, id string) (ImportSession, error) {
|
|
var x ImportSession
|
|
err := s.DB.QueryRow(`SELECT uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms FROM import_sessions WHERE domain_uuid=`+s.sqlParam(1)+` AND uuid=`+s.sqlParam(2)+``, domainUUID, id).Scan(&x.UUID, &x.DomainUUID, &x.ActorUUID, &x.ArchiveName, &x.ArchivePath, &x.TargetPrefix, &x.Status, &x.TotalFiles, &x.MarkdownFiles, &x.Error, &x.CreatedAtMS, &x.UpdatedAtMS)
|
|
return x, err
|
|
}
|
|
|
|
func (s *Store) ListImportSessions(domainUUID string) ([]ImportSession, error) {
|
|
rows, err := s.DB.Query(`SELECT uuid,domain_uuid,actor_uuid,archive_name,archive_path,target_prefix,status,total_files,markdown_files,error,created_at_ms,updated_at_ms FROM import_sessions WHERE domain_uuid=`+s.sqlParam(1)+` ORDER BY created_at_ms DESC LIMIT 50`, domainUUID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []ImportSession
|
|
for rows.Next() {
|
|
var x ImportSession
|
|
if err := rows.Scan(&x.UUID, &x.DomainUUID, &x.ActorUUID, &x.ArchiveName, &x.ArchivePath, &x.TargetPrefix, &x.Status, &x.TotalFiles, &x.MarkdownFiles, &x.Error, &x.CreatedAtMS, &x.UpdatedAtMS); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, x)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *Store) ReplaceImportItems(sessionUUID string, items []ImportItem) (resultErr error) {
|
|
defer func() { resultErr = s.sqlError("replace import items", resultErr) }()
|
|
tx, cancel, err := s.beginImportMutation(sessionUUID)
|
|
defer cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
if _, err = tx.Exec(`DELETE FROM import_items WHERE session_uuid=`+s.sqlParam(1)+``, sessionUUID); err != nil {
|
|
return err
|
|
}
|
|
markdown := 0
|
|
for _, x := range items {
|
|
if x.ContentType == "text/markdown" {
|
|
markdown++
|
|
}
|
|
if _, err = tx.Exec(`INSERT INTO import_items(session_uuid,sequence,source_path,target_path,size,content_type,title,suggested_quartier,suggested_street,suggested_spot,confidence) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`,`+s.sqlParam(9)+`,`+s.sqlParam(10)+`,`+s.sqlParam(11)+`)`, sessionUUID, x.Sequence, x.SourcePath, x.TargetPath, x.Size, x.ContentType, x.Title, x.Quartier, x.Street, x.Spot, x.Confidence); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = tx.Exec(`UPDATE import_sessions SET status='ready',total_files=`+s.sqlParam(1)+`,markdown_files=`+s.sqlParam(2)+`,error='',updated_at_ms=`+s.sqlParam(3)+` WHERE uuid=`+s.sqlParam(4)+``, len(items), markdown, time.Now().UnixMilli(), sessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
func (s *Store) FailImportSession(id string, failure error) {
|
|
_, _ = s.execImportMutation(id, `UPDATE import_sessions SET status='failed',error=`+s.sqlParam(1)+`,updated_at_ms=`+s.sqlParam(2)+` WHERE uuid=`+s.sqlParam(3)+``, failure.Error(), time.Now().UnixMilli(), id)
|
|
}
|
|
func (s *Store) MarkImportAnalyzing(id string) {
|
|
_, _ = s.execImportMutation(id, `UPDATE import_sessions SET status='analyzing',updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+``, time.Now().UnixMilli(), id)
|
|
}
|
|
|
|
func (s *Store) ListImportItems(sessionUUID string) ([]ImportItem, error) {
|
|
rows, err := s.DB.Query(`SELECT i.sequence,i.source_path,i.target_path,i.size,i.content_type,i.title,i.suggested_quartier,i.suggested_street,i.suggested_spot,i.confidence,i.selected,COALESCE(j.completed,0) FROM import_items i LEFT JOIN import_journal j ON j.session_uuid=i.session_uuid AND j.sequence=i.sequence WHERE i.session_uuid=`+s.sqlParam(1)+` ORDER BY i.sequence`, sessionUUID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []ImportItem
|
|
for rows.Next() {
|
|
var x ImportItem
|
|
var selected, imported int
|
|
if err := rows.Scan(&x.Sequence, &x.SourcePath, &x.TargetPath, &x.Size, &x.ContentType, &x.Title, &x.Quartier, &x.Street, &x.Spot, &x.Confidence, &selected, &imported); err != nil {
|
|
return nil, err
|
|
}
|
|
x.Selected = selected != 0
|
|
x.Imported = imported != 0
|
|
if x.ContentType == "text/markdown" {
|
|
x.TargetPath = strings.TrimSuffix(strings.TrimSuffix(x.TargetPath, ".md"), ".markdown")
|
|
}
|
|
out = append(out, x)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *Store) UpdateImportPlan(domainUUID, sessionUUID string, items []ImportItem) (resultErr error) {
|
|
defer func() { resultErr = s.sqlError("update import plan", resultErr) }()
|
|
tx, cancel, err := s.beginPathMutation(domainUUID)
|
|
defer cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var status string
|
|
if err := tx.QueryRow(`SELECT status FROM import_sessions WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2)+``, sessionUUID, domainUUID).Scan(&status); err != nil {
|
|
return err
|
|
}
|
|
if status != "ready" && status != "failed" {
|
|
return namespaceValidationError("import plan is not editable")
|
|
}
|
|
for _, x := range items {
|
|
x.Quartier, x.Street, x.Spot = strings.TrimSpace(x.Quartier), strings.TrimSpace(x.Street), strings.TrimSpace(x.Spot)
|
|
if x.Street != "" && x.Quartier == "" {
|
|
return namespaceValidationError("Street requires a Quartier")
|
|
}
|
|
if x.Spot != "" && x.Street == "" {
|
|
return namespaceValidationError("Spot requires a Street")
|
|
}
|
|
if x.Spot == x.Street {
|
|
x.Spot = ""
|
|
}
|
|
for _, name := range []string{x.Quartier, x.Street, x.Spot} {
|
|
if name != "" {
|
|
if _, err := normalizeCityName(name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
if strings.TrimSpace(x.TargetPath) == "" || normalizePath(x.TargetPath) == "/" {
|
|
return namespaceValidationError("import target Path must identify a document")
|
|
}
|
|
var contentType string
|
|
if err := tx.QueryRow(`SELECT content_type FROM import_items WHERE session_uuid=`+s.sqlParam(1)+` AND sequence=`+s.sqlParam(2)+``, sessionUUID, x.Sequence).Scan(&contentType); err != nil {
|
|
return err
|
|
}
|
|
target := normalizePath(x.TargetPath)
|
|
if contentType == "text/markdown" {
|
|
target = strings.TrimSuffix(strings.TrimSuffix(target, ".md"), ".markdown")
|
|
}
|
|
result, err := tx.Exec(`UPDATE import_items SET target_path=`+s.sqlParam(1)+`,suggested_quartier=`+s.sqlParam(2)+`,suggested_street=`+s.sqlParam(3)+`,suggested_spot=`+s.sqlParam(4)+`,selected=`+s.sqlParam(5)+` WHERE session_uuid=`+s.sqlParam(6)+` AND sequence=`+s.sqlParam(7)+` AND NOT EXISTS(SELECT 1 FROM import_journal j WHERE j.session_uuid=import_items.session_uuid AND j.sequence=import_items.sequence AND j.completed=1)`, target, x.Quartier, x.Street, x.Spot, boolToInt(x.Selected), sessionUUID, x.Sequence)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := result.RowsAffected(); n != 1 {
|
|
continue
|
|
}
|
|
}
|
|
_, err = tx.Exec(`UPDATE import_sessions SET updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+``, time.Now().UnixMilli(), sessionUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
func (s *Store) QueueImportCommit(domainUUID, sessionUUID string) error {
|
|
result, err := s.execImportMutation(sessionUUID, `UPDATE import_sessions SET status='queued_import',error='',updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+` AND domain_uuid=`+s.sqlParam(3)+` AND status IN ('ready','failed')`, time.Now().UnixMilli(), sessionUUID, domainUUID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := result.RowsAffected(); n != 1 {
|
|
return namespaceValidationError("import is not ready")
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Store) MarkImportStatus(id, status string) {
|
|
_, _ = s.execImportMutation(id, `UPDATE import_sessions SET status=`+s.sqlParam(1)+`,updated_at_ms=`+s.sqlParam(2)+` WHERE uuid=`+s.sqlParam(3)+``, status, time.Now().UnixMilli(), id)
|
|
}
|
|
func (s *Store) AddImportJournal(sessionUUID string, sequence int, path, objectUUID string) error {
|
|
_, err := s.execImportMutation(sessionUUID, `INSERT INTO import_journal(session_uuid,sequence,path,object_uuid,created_at_ms,completed) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,0)`, sessionUUID, sequence, path, objectUUID, time.Now().UnixMilli())
|
|
return err
|
|
}
|
|
func (s *Store) CompleteImportJournal(sessionUUID string, sequence int) error {
|
|
_, err := s.execImportMutation(sessionUUID, `UPDATE import_journal SET completed=1 WHERE session_uuid=`+s.sqlParam(1)+` AND sequence=`+s.sqlParam(2)+``, sessionUUID, sequence)
|
|
return err
|
|
}
|
|
func (s *Store) AddImportCityJournal(sessionUUID, kind, nodeUUID string) error {
|
|
_, err := s.execImportMutation(sessionUUID, `INSERT INTO import_city_journal(session_uuid,kind,node_uuid,created_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`) ON CONFLICT DO NOTHING`, sessionUUID, kind, nodeUUID, time.Now().UnixMilli())
|
|
return err
|
|
}
|
|
func (s *Store) AddImportCityEdge(sessionUUID, spotUUID, pageUUID string) error {
|
|
_, err := s.execImportMutation(sessionUUID, `INSERT INTO import_city_edges(session_uuid,spot_uuid,page_uuid,created_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,`+s.sqlParam(3)+`,`+s.sqlParam(4)+`) ON CONFLICT DO NOTHING`, sessionUUID, spotUUID, pageUUID, time.Now().UnixMilli())
|
|
return err
|
|
}
|
|
|
|
type ImportCityJournalEntry struct{ Kind, UUID string }
|
|
|
|
func (s *Store) ImportCityJournal(sessionUUID string) ([]ImportCityJournalEntry, error) {
|
|
rows, err := s.DB.Query(`SELECT kind,node_uuid FROM import_city_journal WHERE session_uuid=`+s.sqlParam(1)+` ORDER BY CASE kind WHEN 'spot' THEN 1 WHEN 'street' THEN 2 ELSE 3 END`, sessionUUID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []ImportCityJournalEntry
|
|
for rows.Next() {
|
|
var x ImportCityJournalEntry
|
|
if err := rows.Scan(&x.Kind, &x.UUID); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, x)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
type ImportCityEdge struct{ SpotUUID, PageUUID string }
|
|
|
|
func (s *Store) ImportCityEdges(sessionUUID string) ([]ImportCityEdge, error) {
|
|
rows, err := s.DB.Query(`SELECT spot_uuid,page_uuid FROM import_city_edges WHERE session_uuid=`+s.sqlParam(1)+``, sessionUUID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []ImportCityEdge
|
|
for rows.Next() {
|
|
var x ImportCityEdge
|
|
if err := rows.Scan(&x.SpotUUID, &x.PageUUID); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, x)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
type ImportJournalEntry struct {
|
|
Sequence int
|
|
Path, ObjectUUID string
|
|
Completed bool
|
|
}
|
|
|
|
func (s *Store) ImportJournal(sessionUUID string) ([]ImportJournalEntry, error) {
|
|
rows, err := s.DB.Query(`SELECT sequence,path,object_uuid,completed FROM import_journal WHERE session_uuid=`+s.sqlParam(1)+` ORDER BY sequence`, sessionUUID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []ImportJournalEntry
|
|
for rows.Next() {
|
|
var x ImportJournalEntry
|
|
var completed int
|
|
if err := rows.Scan(&x.Sequence, &x.Path, &x.ObjectUUID, &completed); err != nil {
|
|
return nil, err
|
|
}
|
|
x.Completed = completed != 0
|
|
out = append(out, x)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// RollbackImportPaths removes only bindings still owned by this import. A Path
|
|
// subsequently rebound to another Object must survive. City cleanup is a
|
|
// separate, retryable phase performed by the caller before this transaction.
|
|
func (s *Store) RollbackImportPaths(domain, session string) (resultErr error) {
|
|
defer func() { resultErr = s.sqlError("rollback import paths", resultErr) }()
|
|
tx, cancel, err := s.beginPathMutation(domain)
|
|
defer cancel()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback()
|
|
var status string
|
|
if err = tx.QueryRow(`SELECT status FROM import_sessions WHERE uuid=`+s.sqlParam(1)+` AND domain_uuid=`+s.sqlParam(2), session, domain).Scan(&status); err != nil {
|
|
return err
|
|
}
|
|
if status != "completed" && status != "failed" {
|
|
return namespaceValidationError("only a completed or failed import can be rolled back")
|
|
}
|
|
if _, err = tx.Exec(`DELETE FROM paths WHERE domain_uuid=`+s.sqlParam(1)+` AND EXISTS(SELECT 1 FROM import_journal j WHERE j.session_uuid=`+s.sqlParam(2)+` AND j.path=paths.path AND j.object_uuid=paths.object_uuid)`, domain, session); err != nil {
|
|
return err
|
|
}
|
|
if _, err = tx.Exec(`UPDATE import_sessions SET status='rolled_back',updated_at_ms=`+s.sqlParam(1)+` WHERE uuid=`+s.sqlParam(2)+` AND domain_uuid=`+s.sqlParam(3), time.Now().UnixMilli(), session, domain); err != nil {
|
|
return err
|
|
}
|
|
if err = tx.Commit(); err == nil {
|
|
s.InvalidatePathCaches()
|
|
}
|
|
return err
|
|
}
|