feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
+105
-17
@@ -1,12 +1,16 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/metadata"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
@@ -48,6 +52,14 @@ type AssetDependency struct {
|
||||
}
|
||||
|
||||
func (s *Store) EnsureAssetsSchema() error {
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
// Native core migrations own these tables; never run SQLite adoption.
|
||||
rows, err := s.DB.Query(`SELECT a.domain_uuid,a.address,a.object_uuid,a.head_document_hash,a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,h.id,h.action,h.document_hash,h.actor_uuid FROM assets a JOIN asset_head_history h ON h.domain_uuid=a.domain_uuid AND h.address=a.address WHERE 1=0`)
|
||||
if err != nil {
|
||||
return s.sqlError("asset schema", err)
|
||||
}
|
||||
return rows.Close()
|
||||
}
|
||||
columns, err := assetTableColumns(s.DB, "assets")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -129,8 +141,9 @@ func assetTableColumns(db *sql.DB, table string) (map[string]bool, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ListAssets(domainUUID string) ([]Asset, error) {
|
||||
rows, err := s.DB.Query(`SELECT a.domain_uuid,a.address,a.object_uuid,COALESCE(a.head_document_hash,''),a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,(SELECT COUNT(DISTINCT x.revision_uuid) FROM revision_document_refs x JOIN revisions r ON r.uuid=x.revision_uuid WHERE x.document_hash IN (SELECT document_hash FROM asset_head_history h WHERE h.domain_uuid=a.domain_uuid AND h.address=a.address UNION SELECT a.head_document_hash) AND EXISTS(SELECT 1 FROM paths p WHERE p.domain_uuid=a.domain_uuid AND p.object_uuid=r.object_uuid)) FROM assets a WHERE a.domain_uuid=? ORDER BY a.updated_at_ms DESC,a.address`, domainUUID)
|
||||
func (s *Store) ListAssets(domainUUID string) (_ []Asset, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("list assets", resultErr) }()
|
||||
rows, err := s.DB.Query(`SELECT a.domain_uuid,a.address,a.object_uuid,COALESCE(a.head_document_hash,''),a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,(SELECT COUNT(DISTINCT x.revision_uuid) FROM revision_document_refs x JOIN revisions r ON r.uuid=x.revision_uuid WHERE x.document_hash IN (SELECT document_hash FROM asset_head_history h WHERE h.domain_uuid=a.domain_uuid AND h.address=a.address UNION SELECT a.head_document_hash) AND EXISTS(SELECT 1 FROM paths p WHERE p.domain_uuid=a.domain_uuid AND p.object_uuid=r.object_uuid)) FROM assets a WHERE a.domain_uuid=`+s.sqlParam(1)+` ORDER BY a.updated_at_ms DESC,a.address`, domainUUID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -146,22 +159,24 @@ func (s *Store) ListAssets(domainUUID string) ([]Asset, error) {
|
||||
return assets, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) GetAsset(domainUUID, address string) (*Asset, error) {
|
||||
func (s *Store) GetAsset(domainUUID, address string) (_ *Asset, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("get asset", resultErr) }()
|
||||
var a Asset
|
||||
err := s.DB.QueryRow(`SELECT a.domain_uuid,a.address,a.object_uuid,COALESCE(a.head_document_hash,''),a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,(SELECT COUNT(DISTINCT x.revision_uuid) FROM revision_document_refs x JOIN revisions r ON r.uuid=x.revision_uuid WHERE x.document_hash IN (SELECT document_hash FROM asset_head_history h WHERE h.domain_uuid=a.domain_uuid AND h.address=a.address UNION SELECT a.head_document_hash) AND EXISTS(SELECT 1 FROM paths p WHERE p.domain_uuid=a.domain_uuid AND p.object_uuid=r.object_uuid)) FROM assets a WHERE a.domain_uuid=? AND a.address=?`, domainUUID, address).Scan(&a.DomainUUID, &a.Address, &a.ObjectUUID, &a.HeadDocumentHash, &a.ContentType, &a.Filename, &a.SizeBytes, &a.CreatedAtMS, &a.UpdatedAtMS, &a.ReferenceCount)
|
||||
err := s.DB.QueryRow(`SELECT a.domain_uuid,a.address,a.object_uuid,COALESCE(a.head_document_hash,''),a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,(SELECT COUNT(DISTINCT x.revision_uuid) FROM revision_document_refs x JOIN revisions r ON r.uuid=x.revision_uuid WHERE x.document_hash IN (SELECT document_hash FROM asset_head_history h WHERE h.domain_uuid=a.domain_uuid AND h.address=a.address UNION SELECT a.head_document_hash) AND EXISTS(SELECT 1 FROM paths p WHERE p.domain_uuid=a.domain_uuid AND p.object_uuid=r.object_uuid)) FROM assets a WHERE a.domain_uuid=`+s.sqlParam(1)+` AND a.address=`+s.sqlParam(2), domainUUID, address).Scan(&a.DomainUUID, &a.Address, &a.ObjectUUID, &a.HeadDocumentHash, &a.ContentType, &a.Filename, &a.SizeBytes, &a.CreatedAtMS, &a.UpdatedAtMS, &a.ReferenceCount)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrAssetNotFound
|
||||
}
|
||||
return &a, err
|
||||
}
|
||||
|
||||
func (s *Store) GetAssetByHeadHash(domainUUID, hash string) (*Asset, error) {
|
||||
func (s *Store) GetAssetByHeadHash(domainUUID, hash string) (_ *Asset, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("get asset hash", resultErr) }()
|
||||
hash = strings.ToLower(strings.TrimSpace(hash))
|
||||
if !validDocumentHash(hash) {
|
||||
return nil, ErrAssetNotFound
|
||||
}
|
||||
var a Asset
|
||||
err := s.DB.QueryRow(`SELECT a.domain_uuid,a.address,a.object_uuid,COALESCE(a.head_document_hash,''),a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,(SELECT COUNT(DISTINCT x.revision_uuid) FROM revision_document_refs x JOIN revisions r ON r.uuid=x.revision_uuid WHERE x.document_hash IN (SELECT document_hash FROM asset_head_history h WHERE h.domain_uuid=a.domain_uuid AND h.address=a.address UNION SELECT a.head_document_hash) AND EXISTS(SELECT 1 FROM paths p WHERE p.domain_uuid=a.domain_uuid AND p.object_uuid=r.object_uuid)) FROM assets a WHERE a.domain_uuid=? AND a.head_document_hash=? LIMIT 1`, domainUUID, hash).Scan(&a.DomainUUID, &a.Address, &a.ObjectUUID, &a.HeadDocumentHash, &a.ContentType, &a.Filename, &a.SizeBytes, &a.CreatedAtMS, &a.UpdatedAtMS, &a.ReferenceCount)
|
||||
err := s.DB.QueryRow(`SELECT a.domain_uuid,a.address,a.object_uuid,COALESCE(a.head_document_hash,''),a.content_type,a.filename,a.size_bytes,a.created_at_ms,a.updated_at_ms,(SELECT COUNT(DISTINCT x.revision_uuid) FROM revision_document_refs x JOIN revisions r ON r.uuid=x.revision_uuid WHERE x.document_hash IN (SELECT document_hash FROM asset_head_history h WHERE h.domain_uuid=a.domain_uuid AND h.address=a.address UNION SELECT a.head_document_hash) AND EXISTS(SELECT 1 FROM paths p WHERE p.domain_uuid=a.domain_uuid AND p.object_uuid=r.object_uuid)) FROM assets a WHERE a.domain_uuid=`+s.sqlParam(1)+` AND a.head_document_hash=`+s.sqlParam(2)+` LIMIT 1`, domainUUID, hash).Scan(&a.DomainUUID, &a.Address, &a.ObjectUUID, &a.HeadDocumentHash, &a.ContentType, &a.Filename, &a.SizeBytes, &a.CreatedAtMS, &a.UpdatedAtMS, &a.ReferenceCount)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrAssetNotFound
|
||||
}
|
||||
@@ -178,6 +193,12 @@ func normalizeAssetAddress(address string) (string, error) {
|
||||
}
|
||||
|
||||
func (s *Store) PutAsset(domainUUID, address string, payload []byte, contentType, filename, expectedHash, actorUUID string) (*Asset, bool, error) {
|
||||
if s.ReadOnly {
|
||||
return nil, false, ErrReadOnly
|
||||
}
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
return s.putNativeAsset(domainUUID, address, payload, contentType, filename, expectedHash, actorUUID)
|
||||
}
|
||||
if s.DocumentRoot == "" {
|
||||
return nil, false, errors.New("Asset writes require a configured Document Store")
|
||||
}
|
||||
@@ -244,8 +265,9 @@ func (s *Store) PutAsset(domainUUID, address string, payload []byte, contentType
|
||||
return a, true, err
|
||||
}
|
||||
|
||||
func (s *Store) ListAssetHistory(domainUUID, address string) ([]AssetHistory, error) {
|
||||
rows, err := s.DB.Query(`SELECT id,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms FROM asset_head_history WHERE domain_uuid=? AND address=? ORDER BY id DESC LIMIT 100`, domainUUID, address)
|
||||
func (s *Store) ListAssetHistory(domainUUID, address string) (_ []AssetHistory, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("asset history", resultErr) }()
|
||||
rows, err := s.DB.Query(`SELECT id,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms FROM asset_head_history WHERE domain_uuid=`+s.sqlParam(1)+` AND address=`+s.sqlParam(2)+` ORDER BY id DESC LIMIT 100`, domainUUID, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -261,7 +283,8 @@ func (s *Store) ListAssetHistory(domainUUID, address string) ([]AssetHistory, er
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ListAssetDependencies(domainUUID, address string) ([]AssetDependency, error) {
|
||||
func (s *Store) ListAssetDependencies(domainUUID, address string) (_ []AssetDependency, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("asset dependencies", resultErr) }()
|
||||
rows, err := s.DB.Query(`
|
||||
SELECT DISTINCT x.document_hash,r.uuid,r.object_uuid,p.path,
|
||||
COALESCE(r.created_at_ms,COALESCE(r.created_at,0)*1000)
|
||||
@@ -270,7 +293,7 @@ func (s *Store) ListAssetDependencies(domainUUID, address string) ([]AssetDepend
|
||||
JOIN revision_document_refs x ON x.document_hash=h.document_hash
|
||||
JOIN revisions r ON r.uuid=x.revision_uuid
|
||||
JOIN paths p ON p.domain_uuid=h.domain_uuid AND p.object_uuid=r.object_uuid
|
||||
WHERE h.domain_uuid=? AND h.address=?
|
||||
WHERE h.domain_uuid=`+s.sqlParam(1)+` AND h.address=`+s.sqlParam(2)+`
|
||||
ORDER BY COALESCE(r.created_at_ms,COALESCE(r.created_at,0)*1000) DESC,p.path
|
||||
LIMIT 500
|
||||
`, domainUUID, address)
|
||||
@@ -290,6 +313,9 @@ func (s *Store) ListAssetDependencies(domainUUID, address string) ([]AssetDepend
|
||||
}
|
||||
|
||||
func (s *Store) RestoreAssetHead(domainUUID, address string, historyID int64, expectedHash, actorUUID string) (*Asset, error) {
|
||||
if s.ReadOnly {
|
||||
return nil, ErrReadOnly
|
||||
}
|
||||
current, err := s.GetAsset(domainUUID, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -298,25 +324,40 @@ func (s *Store) RestoreAssetHead(domainUUID, address string, historyID int64, ex
|
||||
return nil, ErrAssetConflict
|
||||
}
|
||||
var historical AssetHistory
|
||||
err = s.DB.QueryRow(`SELECT id,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms FROM asset_head_history WHERE domain_uuid=? AND address=? AND id=?`, domainUUID, address, historyID).Scan(&historical.ID, &historical.Action, &historical.DocumentHash, &historical.ContentType, &historical.Filename, &historical.SizeBytes, &historical.ActorUUID, &historical.CreatedAtMS)
|
||||
err = s.DB.QueryRow(`SELECT id,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms FROM asset_head_history WHERE domain_uuid=`+s.sqlParam(1)+` AND address=`+s.sqlParam(2)+` AND id=`+s.sqlParam(3), domainUUID, address, historyID).Scan(&historical.ID, &historical.Action, &historical.DocumentHash, &historical.ContentType, &historical.Filename, &historical.SizeBytes, &historical.ActorUUID, &historical.CreatedAtMS)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, ErrAssetNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
if s.DocumentRoot == "" {
|
||||
return nil, ErrDocumentStoreRequired
|
||||
}
|
||||
docs, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err = docs.Verify(ctx, historical.DocumentHash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
reader, err := s.OpenDocument(historical.DocumentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = reader.Close()
|
||||
now := time.Now().UnixMilli()
|
||||
tx, err := s.DB.Begin()
|
||||
tx, cancel, err := s.beginPathMutation(domainUUID)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.Exec(`UPDATE assets SET head_document_hash=?,content_type=?,filename=?,size_bytes=?,updated_at_ms=? WHERE domain_uuid=? AND address=? AND head_document_hash=?`, historical.DocumentHash, historical.ContentType, historical.Filename, historical.SizeBytes, now, domainUUID, address, current.HeadDocumentHash)
|
||||
result, err := tx.Exec(`UPDATE assets SET head_document_hash=`+s.sqlParam(1)+`,content_type=`+s.sqlParam(2)+`,filename=`+s.sqlParam(3)+`,size_bytes=`+s.sqlParam(4)+`,updated_at_ms=`+s.sqlParam(5)+` WHERE domain_uuid=`+s.sqlParam(6)+` AND address=`+s.sqlParam(7)+` AND head_document_hash=`+s.sqlParam(8), historical.DocumentHash, historical.ContentType, historical.Filename, historical.SizeBytes, now, domainUUID, address, current.HeadDocumentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -324,7 +365,7 @@ func (s *Store) RestoreAssetHead(domainUUID, address string, historyID int64, ex
|
||||
if affected != 1 {
|
||||
return nil, ErrAssetConflict
|
||||
}
|
||||
if _, err = tx.Exec(`INSERT INTO asset_head_history(domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms) VALUES(?,?,'restore',?,?,?,?,?,?)`, domainUUID, address, historical.DocumentHash, historical.ContentType, historical.Filename, historical.SizeBytes, actorUUID, now); err != nil {
|
||||
if _, err = tx.Exec(`INSERT INTO asset_head_history(domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,'restore',`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`)`, domainUUID, address, historical.DocumentHash, historical.ContentType, historical.Filename, historical.SizeBytes, actorUUID, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
@@ -334,6 +375,9 @@ func (s *Store) RestoreAssetHead(domainUUID, address string, historyID int64, ex
|
||||
}
|
||||
|
||||
func (s *Store) DeleteAsset(domainUUID, address, expectedHash, actorUUID string) error {
|
||||
if s.ReadOnly {
|
||||
return ErrReadOnly
|
||||
}
|
||||
a, err := s.GetAsset(domainUUID, address)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -342,15 +386,16 @@ func (s *Store) DeleteAsset(domainUUID, address, expectedHash, actorUUID string)
|
||||
return ErrAssetConflict
|
||||
}
|
||||
now := time.Now().UnixMilli()
|
||||
tx, err := s.DB.Begin()
|
||||
tx, cancel, err := s.beginPathMutation(domainUUID)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err = tx.Exec(`INSERT INTO asset_head_history(domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms) VALUES(?,?,'delete',?,?,?,?,?,?)`, domainUUID, address, a.HeadDocumentHash, a.ContentType, a.Filename, a.SizeBytes, actorUUID, now); err != nil {
|
||||
if _, err = tx.Exec(`INSERT INTO asset_head_history(domain_uuid,address,action,document_hash,content_type,filename,size_bytes,actor_uuid,created_at_ms) VALUES(`+s.sqlParam(1)+`,`+s.sqlParam(2)+`,'delete',`+s.sqlParam(3)+`,`+s.sqlParam(4)+`,`+s.sqlParam(5)+`,`+s.sqlParam(6)+`,`+s.sqlParam(7)+`,`+s.sqlParam(8)+`)`, domainUUID, address, a.HeadDocumentHash, a.ContentType, a.Filename, a.SizeBytes, actorUUID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := tx.Exec(`DELETE FROM assets WHERE domain_uuid=? AND address=? AND head_document_hash=?`, domainUUID, address, a.HeadDocumentHash)
|
||||
result, err := tx.Exec(`DELETE FROM assets WHERE domain_uuid=`+s.sqlParam(1)+` AND address=`+s.sqlParam(2)+` AND head_document_hash=`+s.sqlParam(3), domainUUID, address, a.HeadDocumentHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -359,3 +404,46 @@ func (s *Store) DeleteAsset(domainUUID, address, expectedHash, actorUUID string)
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) putNativeAsset(domain, address string, payload []byte, contentType, filename, expected, actor string) (*Asset, bool, error) {
|
||||
if s.DocumentRoot == "" {
|
||||
return nil, false, ErrDocumentStoreRequired
|
||||
}
|
||||
address, err := normalizeAssetAddress(address)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if strings.TrimSpace(contentType) == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
repo, err := metadata.NewApplication(s.DB, metadata.PostgreSQL)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
docs, err := s.documentStorage()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hash := revisionHash(string(payload))
|
||||
previous, getErr := repo.GetAsset(ctx, domain, address)
|
||||
if getErr != nil && !errors.Is(getErr, metadata.ErrNotFound) {
|
||||
return nil, false, getErr
|
||||
}
|
||||
if previous.HeadDocumentHash != strings.Trim(expected, "\"") {
|
||||
return nil, false, ErrAssetConflict
|
||||
}
|
||||
if err = docs.PutIfAbsent(ctx, hash, int64(len(payload)), bytes.NewReader(payload)); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
a, err := repo.PutAsset(ctx, metadata.Asset{DomainUUID: domain, Address: address, HeadDocumentHash: hash, ContentType: contentType, Filename: strings.TrimSpace(filename), SizeBytes: int64(len(payload))}, strings.Trim(expected, "\""), actor)
|
||||
if errors.Is(err, metadata.ErrConflict) {
|
||||
return nil, false, ErrAssetConflict
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
result, err := s.GetAsset(domain, address)
|
||||
return result, a.UpdatedAtMS != previous.UpdatedAtMS, err
|
||||
}
|
||||
Reference in new issue
Block a user