feat(storage): add native PostgreSQL runtime and verified offline application import
Build and publish container images / images (push) Failing after 12m29s
Build and publish container images / images (push) Failing after 12m29s
This commit is contained in:
1 parent
96065d1e89
commit
0ba1dcfef0
243 files changed
+21441
-3090
No files matched your search
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
@@ -30,20 +31,23 @@ func (s *Store) EnsureProjectionMetadataSchema() error {
|
||||
return s.ensureSchemaComponent("projection-metadata")
|
||||
}
|
||||
|
||||
func (s *Store) GetProjectionMetadata(hash, kind string, version int) (*ProjectionMetadata, error) {
|
||||
func (s *Store) GetProjectionMetadata(hash, kind string, version int) (_ *ProjectionMetadata, resultErr error) {
|
||||
defer func() { resultErr = s.sqlError("projection metadata", resultErr) }()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if !validDocumentHash(hash) || version < 1 || (kind != "pdf" && kind != "archive") {
|
||||
return nil, errors.New("invalid projection metadata key")
|
||||
}
|
||||
entry := &ProjectionMetadata{DocumentHash: hash, Kind: kind, Version: version}
|
||||
var payload string
|
||||
err := s.DB.QueryRow(`SELECT payload,error,updated_at_ms,retry_at_ms FROM projection_metadata
|
||||
WHERE document_hash=? AND kind=? AND version=?`, hash, kind, version).
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT payload,error,updated_at_ms,retry_at_ms FROM projection_metadata
|
||||
WHERE document_hash=`+s.sqlParam(1)+` AND kind=`+s.sqlParam(2)+` AND version=`+s.sqlParam(3), hash, kind, version).
|
||||
Scan(&payload, &entry.Error, &entry.UpdatedAtMS, &entry.RetryAtMS)
|
||||
entry.Payload = json.RawMessage(payload)
|
||||
return entry, err
|
||||
}
|
||||
|
||||
func (s *Store) SaveProjectionMetadata(entry ProjectionMetadata) error {
|
||||
func (s *Store) SaveProjectionMetadata(entry ProjectionMetadata) (resultErr error) {
|
||||
if !validDocumentHash(entry.DocumentHash) || entry.Version < 1 ||
|
||||
(entry.Kind != "pdf" && entry.Kind != "archive") ||
|
||||
len(entry.Payload) > MaxProjectionMetadataBytes || !json.Valid(entry.Payload) {
|
||||
@@ -55,27 +59,37 @@ func (s *Store) SaveProjectionMetadata(entry ProjectionMetadata) error {
|
||||
if entry.UpdatedAtMS == 0 {
|
||||
entry.UpdatedAtMS = time.Now().UnixMilli()
|
||||
}
|
||||
tx, err := s.DB.Begin()
|
||||
defer func() { resultErr = s.sqlError("projection metadata", resultErr) }()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = s.lockTaskWriter(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(`INSERT INTO projection_metadata
|
||||
(document_hash,kind,version,payload,error,updated_at_ms,retry_at_ms)
|
||||
VALUES(?,?,?,?,?,?,?) ON CONFLICT(document_hash,kind,version) DO UPDATE SET
|
||||
VALUES(`+s.sqlValues(7)+`) ON CONFLICT(document_hash,kind,version) DO UPDATE SET
|
||||
payload=excluded.payload,error=excluded.error,updated_at_ms=excluded.updated_at_ms,retry_at_ms=excluded.retry_at_ms`,
|
||||
entry.DocumentHash, entry.Kind, entry.Version, string(entry.Payload), entry.Error, entry.UpdatedAtMS, entry.RetryAtMS); err != nil {
|
||||
return err
|
||||
}
|
||||
// Bound both artifact size and aggregate storage. Eviction happens only on
|
||||
// background writes, not on foreground cache reads. Cache loss is recoverable.
|
||||
byteLength := `length(CAST(payload AS BLOB))+length(CAST(error AS BLOB))`
|
||||
if s.sqlBackend == storePostgreSQL {
|
||||
byteLength = `octet_length(payload)+octet_length(error)`
|
||||
}
|
||||
if _, err = tx.Exec(`DELETE FROM projection_metadata WHERE (document_hash,kind,version) IN (
|
||||
SELECT document_hash,kind,version FROM (
|
||||
SELECT document_hash,kind,version,
|
||||
ROW_NUMBER() OVER(ORDER BY updated_at_ms DESC,document_hash,kind,version) AS rank,
|
||||
SUM(length(CAST(payload AS BLOB))+length(CAST(error AS BLOB))+128)
|
||||
SUM(`+byteLength+`+128)
|
||||
OVER(ORDER BY updated_at_ms DESC,document_hash,kind,version) AS total_bytes
|
||||
FROM projection_metadata) WHERE rank>? OR total_bytes>?)`, maxProjectionCacheRecords, maxProjectionCacheBytes); err != nil {
|
||||
FROM projection_metadata) AS cache_budget WHERE rank>`+s.sqlParam(1)+` OR total_bytes>`+s.sqlParam(2)+`)`, maxProjectionCacheRecords, maxProjectionCacheBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
|
||||
Reference in new issue
Block a user