feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -0,0 +1,257 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pageswiki/internal/documentrefs"
|
||||
"pageswiki/internal/geduldh"
|
||||
)
|
||||
|
||||
type faultDocuments struct {
|
||||
geduldh.Store
|
||||
putErr error
|
||||
verifyErr error
|
||||
afterPut func()
|
||||
}
|
||||
|
||||
func (f faultDocuments) PutIfAbsent(ctx context.Context, h string, n int64, reader io.Reader) error {
|
||||
if f.putErr != nil {
|
||||
return f.putErr
|
||||
}
|
||||
if err := f.Store.PutIfAbsent(ctx, h, n, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.afterPut != nil {
|
||||
f.afterPut()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f faultDocuments) Verify(ctx context.Context, h string) error {
|
||||
if f.verifyErr != nil {
|
||||
return f.verifyErr
|
||||
}
|
||||
return f.Store.Verify(ctx, h)
|
||||
}
|
||||
|
||||
func documentFixture(t *testing.T, r *Repository) (geduldh.Store, string, Revision) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
documents, err := geduldh.NewLocal(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content := []byte("synthetic image")
|
||||
h := hash(string(content))
|
||||
if err := documents.PutIfAbsent(ctx, h, int64(len(content)), bytes.NewReader(content)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.PutAsset(ctx, Asset{DomainUUID: "test", Address: "asset:image:cover", HeadDocumentHash: h, ContentType: "image/png", SizeBytes: int64(len(content))}, "", "actor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
object, err := r.CreatePage(ctx, "test", "/guide")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return documents, h, Revision{ObjectUUID: object, Language: "ja", ContentType: "text/html; charset=utf-8", Message: "synthetic save"}
|
||||
}
|
||||
|
||||
func TestDocumentSaveReferencesOutboxAndRestore(t *testing.T) {
|
||||
for _, backend := range []Backend{SQLite, PostgreSQL} {
|
||||
t.Run(string(backend), func(t *testing.T) {
|
||||
r := testRepository(t, backend)
|
||||
ctx := context.Background()
|
||||
docs, assetHash, v := documentFixture(t, r)
|
||||
body := []byte(`<h1>Guide</h1><img src="asset:image:cover"><img src="asset:image:cover">`)
|
||||
if _, err := r.PutAsset(ctx, Asset{DomainUUID: "private-domain", Address: "asset:image:private", HeadDocumentHash: assetHash, ContentType: "image/png", SizeBytes: 15}, "", "actor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.SaveDocument(ctx, docs, "test", v, []byte(`<img src="asset:image:private">`), "", "actor"); !errors.Is(err, documentrefs.ErrUnavailable) {
|
||||
t.Fatal("foreign Asset resolved", err)
|
||||
}
|
||||
first, err := r.SaveDocument(ctx, docs, "test", v, body, "", "actor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(first.Payload), "/_/obj/sha256:"+assetHash) {
|
||||
t.Fatal("mutable URL not frozen")
|
||||
}
|
||||
if err := docs.Verify(ctx, first.Revision.DocumentHash); err != nil {
|
||||
t.Fatal("metadata published without durable Document", err)
|
||||
}
|
||||
refs, err := r.revisionReferences(ctx, "test", first.Revision.UUID)
|
||||
if err != nil || len(refs) != 1 || refs[0] != assetHash {
|
||||
t.Fatal("embedded reference", err, refs)
|
||||
}
|
||||
var eventDomain, target, payload, actor string
|
||||
var sequence int64
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT domain_uuid,target_uuid,payload,actor_uuid,sequence FROM activity_outbox`).Scan(&eventDomain, &target, &payload, &actor, &sequence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope map[string]string
|
||||
if err := json.Unmarshal([]byte(payload), &envelope); err != nil || eventDomain != "test" || actor != "actor" || sequence != 1 || target != first.Revision.UUID || envelope["document_hash"] != first.Revision.DocumentHash {
|
||||
t.Fatal("outbox envelope mismatch", err)
|
||||
}
|
||||
newContent := []byte("next synthetic image")
|
||||
newHash := hash(string(newContent))
|
||||
if err := docs.PutIfAbsent(ctx, newHash, int64(len(newContent)), bytes.NewReader(newContent)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.PutAsset(ctx, Asset{DomainUUID: "test", Address: "asset:image:cover", HeadDocumentHash: newHash, ContentType: "image/png", SizeBytes: int64(len(newContent))}, assetHash, "actor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := r.SaveDocument(ctx, docs, "test", v, first.Payload, first.Revision.UUID, "actor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(second.Payload), newHash) {
|
||||
t.Fatal("provenance not refreshed")
|
||||
}
|
||||
restored, err := r.RestoreRevision(ctx, "test", first.Revision.UUID, second.Revision.UUID, "restore")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refs, err = r.revisionReferences(ctx, "test", restored.UUID)
|
||||
if err != nil || len(refs) != 1 || refs[0] != assetHash || restored.DocumentHash != first.Revision.DocumentHash {
|
||||
t.Fatal("restore followed new Asset HEAD instead of historical refs", err)
|
||||
}
|
||||
var count int
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM activity_outbox`).Scan(&count); err != nil || count != 3 {
|
||||
t.Fatal("save/restore event count", count, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentSaveFailureIsAtomic(t *testing.T) {
|
||||
for _, backend := range []Backend{SQLite, PostgreSQL} {
|
||||
for _, stage := range []string{"references", "outbox"} {
|
||||
t.Run(string(backend)+"/"+stage, func(t *testing.T) {
|
||||
r := testRepository(t, backend)
|
||||
ctx := context.Background()
|
||||
docs, _, v := documentFixture(t, r)
|
||||
body := []byte(`<img src="asset:image:cover">`)
|
||||
first, err := r.SaveDocument(ctx, docs, "test", v, body, "", "actor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ddl := `CREATE UNIQUE INDEX synthetic_save_failure ON revision_document_refs(document_hash)`
|
||||
if stage == "outbox" {
|
||||
ddl = `CREATE UNIQUE INDEX synthetic_save_failure ON activity_outbox(domain_uuid)`
|
||||
}
|
||||
if _, err := r.database.ExecContext(ctx, ddl); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.SaveDocument(ctx, docs, "test", v, append([]byte(`<p>change</p>`), body...), first.Revision.UUID, "actor"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("injected error ignored", err)
|
||||
}
|
||||
for _, table := range []string{"revisions", "revision_document_refs", "activity_outbox"} {
|
||||
var n int
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM `+table).Scan(&n); err != nil || n != 1 {
|
||||
t.Fatal("partial commit", table, n, err)
|
||||
}
|
||||
}
|
||||
var last int64
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT last_sequence FROM activity_outbox_heads WHERE domain_uuid='test'`).Scan(&last); err != nil || last != 1 {
|
||||
t.Fatal("sequence advanced on failure", err)
|
||||
}
|
||||
if _, err := r.database.ExecContext(ctx, `DROP INDEX synthetic_save_failure`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.SaveDocument(ctx, docs, "test", v, body, first.Revision.UUID, "actor"); err != nil {
|
||||
t.Fatal("retry after rollback", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentSavePersistenceFailureAndConcurrentEdit(t *testing.T) {
|
||||
for _, backend := range []Backend{SQLite, PostgreSQL} {
|
||||
t.Run(string(backend), func(t *testing.T) {
|
||||
r := testRepository(t, backend)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
docs, _, v := documentFixture(t, r)
|
||||
body := []byte(`<img src="asset:image:cover">`)
|
||||
for _, failing := range []faultDocuments{{Store: docs, putErr: io.ErrClosedPipe}, {Store: docs, verifyErr: geduldh.ErrHashMismatch}} {
|
||||
if _, err := r.SaveDocument(ctx, failing, "test", v, body, "", "actor"); err == nil {
|
||||
t.Fatal("Document failure ignored")
|
||||
}
|
||||
var n int
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM revisions`).Scan(&n); err != nil || n != 0 {
|
||||
t.Fatal("failed persistence published revision", err)
|
||||
}
|
||||
}
|
||||
first, err := r.SaveDocument(ctx, docs, "test", v, body, "", "actor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The callback writes using a different transaction during Document I/O.
|
||||
// It would deadlock if SaveDocument held SQL locks over the payload write.
|
||||
race := faultDocuments{Store: docs, afterPut: func() {
|
||||
peer, _ := New(r.database, backend)
|
||||
competing := v
|
||||
competing.DocumentHash = first.Revision.DocumentHash
|
||||
if _, err := peer.AppendRevision(ctx, "test", competing, first.Revision.UUID); err != nil {
|
||||
t.Fatal("I/O held metadata lock", err)
|
||||
}
|
||||
}}
|
||||
if _, err := r.SaveDocument(ctx, race, "test", v, []byte("<p>stale</p>"), first.Revision.UUID, "actor"); !errors.Is(err, ErrConflict) {
|
||||
t.Fatal("CAS not repeated after payload I/O", err)
|
||||
}
|
||||
var n int
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT count(*) FROM activity_outbox`).Scan(&n); err != nil || n != 2 {
|
||||
t.Fatal("conflicting write emitted event", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutboxSequenceForConcurrentObjects(t *testing.T) {
|
||||
for _, backend := range []Backend{SQLite, PostgreSQL} {
|
||||
t.Run(string(backend), func(t *testing.T) {
|
||||
r := testRepository(t, backend)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
objects := make([]string, 6)
|
||||
for i := range objects {
|
||||
var err error
|
||||
objects[i], err = r.CreatePage(ctx, "test", "/"+strings.Repeat("a", i+1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, len(objects))
|
||||
for _, id := range objects {
|
||||
wg.Add(1)
|
||||
go func(id string) {
|
||||
defer wg.Done()
|
||||
peer, _ := New(r.database, backend)
|
||||
_, err := peer.AppendRevision(ctx, "test", Revision{ObjectUUID: id, DocumentHash: hash(id), ContentType: "text/html"}, "")
|
||||
errs <- err
|
||||
}(id)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var count int
|
||||
var min, max int64
|
||||
if err := r.database.QueryRowContext(ctx, `SELECT count(*),min(sequence),max(sequence) FROM activity_outbox`).Scan(&count, &min, &max); err != nil || count != 6 || min != 1 || max != 6 {
|
||||
t.Fatal("invalid concurrent sequence", count, min, max, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user