feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -0,0 +1,140 @@
|
||||
// Package documentrefs freezes mutable Asset references without interpreting
|
||||
// ordinary text or code examples. Resolvers must authorize the source Domain.
|
||||
package documentrefs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
var ErrUnavailable = errors.New("embedded asset is unavailable")
|
||||
|
||||
type Resolver func(context.Context, string) (string, error)
|
||||
|
||||
// Freeze retains original bytes for unchanged tokens and deduplicates hashes
|
||||
// in appearance order. Repeated addresses resolve only once per payload.
|
||||
func Freeze(ctx context.Context, payload []byte, contentType string, resolve Resolver) ([]byte, []string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil || !strings.EqualFold(mediaType, "text/html") {
|
||||
return payload, nil, nil
|
||||
}
|
||||
z := html.NewTokenizer(bytes.NewReader(payload))
|
||||
var output bytes.Buffer
|
||||
seen := map[string]bool{}
|
||||
resolved := map[string]string{}
|
||||
var hashes []string
|
||||
lookup := func(address string) (string, error) {
|
||||
if hash := resolved[address]; hash != "" {
|
||||
return hash, nil
|
||||
}
|
||||
if resolve == nil {
|
||||
return "", ErrUnavailable
|
||||
}
|
||||
hash, err := resolve(ctx, address)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(hash) != 64 {
|
||||
return "", fmt.Errorf("%w: %s", ErrUnavailable, address)
|
||||
}
|
||||
if _, err := hex.DecodeString(hash); err != nil {
|
||||
return "", fmt.Errorf("%w: %s", ErrUnavailable, address)
|
||||
}
|
||||
hash = strings.ToLower(hash)
|
||||
resolved[address] = hash
|
||||
if !seen[hash] {
|
||||
seen[hash] = true
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
return hash, nil
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
typ := z.Next()
|
||||
if typ == html.ErrorToken {
|
||||
if errors.Is(z.Err(), io.EOF) {
|
||||
return output.Bytes(), hashes, nil
|
||||
}
|
||||
return nil, nil, z.Err()
|
||||
}
|
||||
if typ != html.StartTagToken && typ != html.SelfClosingTagToken {
|
||||
output.Write(z.Raw())
|
||||
continue
|
||||
}
|
||||
// Token() may alter the tokenizer buffer. Copy original bytes first.
|
||||
raw := append([]byte(nil), z.Raw()...)
|
||||
token := z.Token()
|
||||
changed := false
|
||||
assetRef := attribute(token.Attr, "asset-ref")
|
||||
legacy := attribute(token.Attr, "immutable-ref")
|
||||
if assetRef == "" && strings.HasPrefix(legacy, "asset:") {
|
||||
assetRef = legacy
|
||||
}
|
||||
if strings.HasPrefix(assetRef, "asset:") {
|
||||
hash, err := lookup(assetRef)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
setAttribute(&token, "asset-ref", assetRef)
|
||||
setAttribute(&token, "immutable-ref", "sha256:"+hash)
|
||||
changed = true
|
||||
}
|
||||
for _, name := range []string{"src", "href", "poster", "data"} {
|
||||
provenance := "data-pageswiki-asset-" + name
|
||||
address := attribute(token.Attr, provenance)
|
||||
if address == "" {
|
||||
candidate := attribute(token.Attr, name)
|
||||
if strings.HasPrefix(candidate, "asset:") {
|
||||
address = candidate
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(address, "asset:") {
|
||||
continue
|
||||
}
|
||||
hash, err := lookup(address)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
setAttribute(&token, provenance, address)
|
||||
setAttribute(&token, name, "/_/obj/sha256:"+hash)
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
output.WriteString(token.String())
|
||||
} else {
|
||||
output.Write(raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func attribute(attrs []html.Attribute, name string) string {
|
||||
for _, a := range attrs {
|
||||
if strings.EqualFold(a.Key, name) {
|
||||
return strings.TrimSpace(a.Val)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func setAttribute(token *html.Token, name, value string) {
|
||||
for i := range token.Attr {
|
||||
if strings.EqualFold(token.Attr[i].Key, name) {
|
||||
token.Attr[i].Key = name
|
||||
token.Attr[i].Val = value
|
||||
return
|
||||
}
|
||||
}
|
||||
token.Attr = append(token.Attr, html.Attribute{Key: name, Val: value})
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package documentrefs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFreezePreservesUnchangedHTML(t *testing.T) {
|
||||
body := `<!DOCTYPE html><!-- sample --><DIV class='A' title="&">日本語<code><img src="asset:image:example"></code><script>const x="asset:image:sample";</script><a HREF='/plain?a=1&b=2'>Link</a></DIV>`
|
||||
got, refs, err := Freeze(context.Background(), []byte(body), "text/html; charset=utf-8", func(context.Context, string) (string, error) { t.Fatal("ordinary text resolved"); return "", nil })
|
||||
if err != nil || string(got) != body || len(refs) != 0 {
|
||||
t.Fatal("unchanged HTML lost bytes", err, string(got))
|
||||
}
|
||||
for _, mime := range []string{"text/plain", "application/octet-stream", "text/html-not-really"} {
|
||||
got, refs, err := Freeze(context.Background(), []byte(`<img src="asset:image:sample">`), mime, nil)
|
||||
if err != nil || len(refs) != 0 || string(got) != `<img src="asset:image:sample">` {
|
||||
t.Fatal("non-HTML modified", mime, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreezeDeduplicatesAndFollowsProvenance(t *testing.T) {
|
||||
hash := strings.Repeat("a", 64)
|
||||
calls := 0
|
||||
resolve := func(context.Context, string) (string, error) { calls++; return hash, nil }
|
||||
body := `<img src="asset:image:a"><video poster="asset:image:a"></video><object data="asset:image:b"></object><a href="asset:image:a">a</a><embedded-object immutable-ref="asset:image:a"></embedded-object>`
|
||||
got, refs, err := Freeze(context.Background(), []byte(body), "text/html", resolve)
|
||||
if err != nil || len(refs) != 1 || refs[0] != hash || calls != 2 {
|
||||
t.Fatal("deduplication", err, refs, calls)
|
||||
}
|
||||
if !strings.Contains(string(got), `immutable-ref="sha256:`+hash+`"`) || !strings.Contains(string(got), `data-pageswiki-asset-src="asset:image:a"`) {
|
||||
t.Fatal("provenance missing")
|
||||
}
|
||||
hash = strings.Repeat("b", 64)
|
||||
calls = 0
|
||||
updated, refs, err := Freeze(context.Background(), got, "text/html", resolve)
|
||||
if err != nil || len(refs) != 1 || refs[0] != hash || calls != 2 || strings.Contains(string(updated), strings.Repeat("a", 64)) {
|
||||
t.Fatal("resave did not follow HEAD", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreezeRejectsMissingInvalidAndCanceled(t *testing.T) {
|
||||
for _, hash := range []string{"", strings.Repeat("x", 64)} {
|
||||
if _, _, err := Freeze(context.Background(), []byte(`<img src="asset:image:a">`), "text/html", func(context.Context, string) (string, error) { return hash, nil }); !errors.Is(err, ErrUnavailable) {
|
||||
t.Fatal("invalid hash accepted", err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, _, err := Freeze(ctx, []byte("text"), "text/plain", nil); !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("cancellation ignored", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user