diff --git a/internal/settingscontract/contract_test.go b/internal/settingscontract/contract_test.go index 4cf2bfd5..ee2d0f50 100644 --- a/internal/settingscontract/contract_test.go +++ b/internal/settingscontract/contract_test.go @@ -1210,6 +1210,40 @@ func TestQuotedNumbersAreRejected(t *testing.T) { } } +// TestRawInvalidUTF8IsRejected covers the other path to the substitution +// TestLoneSurrogatesAreRejectedForEveryType guards. A raw 0xff byte inside a +// quoted string — what an HTTP body carries when a client encodes text in the +// wrong charset — is not an escape, so the surrogate scan never sees it, but +// encoding/json still turns it into U+FFFD and reports success. The value is +// then stored verbatim, which SQLite's json_valid accepts and Postgres jsonb +// refuses. +func TestRawInvalidUTF8IsRejected(t *testing.T) { + for name, schema := range map[string]*ValueSchema{ + "string": {Type: TypeString}, + "object": {Type: TypeObject, SchemaRef: "subtitle-appearance.json"}, + } { + t.Run(name, func(t *testing.T) { + raw := json.RawMessage([]byte{'"', 0xff, '"'}) + if name == "object" { + raw = json.RawMessage(append(append([]byte(`{"fontSize":"`), 0xff), `"}`...)) + } + if err := schema.ValidateValue(raw, objSchemas); err == nil { + t.Fatal("a value with a raw invalid UTF-8 byte was accepted") + } + if _, err := schema.NormalizeValue(raw, objSchemas); err == nil { + t.Fatal("a value with a raw invalid UTF-8 byte was stored") + } + }) + } + + // Valid multi-byte UTF-8 must still pass; the check is on well-formedness, + // not on being ASCII. + if err := (&ValueSchema{Type: TypeString}).ValidateValue( + json.RawMessage(`"日本語 — café 😀"`), nil); err != nil { + t.Errorf("valid multi-byte UTF-8 was rejected: %v", err) + } +} + // TestLoneSurrogatesAreRejectedForEveryType pins the check to the whole // validation path rather than the object branch it started in. A lone surrogate // decodes to U+FFFD on SQLite and is refused outright by Postgres jsonb, so a diff --git a/internal/settingscontract/validate.go b/internal/settingscontract/validate.go index e6bc76be..a80994c2 100644 --- a/internal/settingscontract/validate.go +++ b/internal/settingscontract/validate.go @@ -10,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "unicode/utf8" "github.com/santhosh-tekuri/jsonschema/v6" ) @@ -389,10 +390,19 @@ func (v *ValueSchema) ValidateValue(raw json.RawMessage, objectSchemas map[strin return errors.New("null is not allowed for this setting") } - // Every type that can carry a string, not just the object branch: a lone - // surrogate in a plain string setting (ui.custom_css) decodes to U+FFFD on - // SQLite and is rejected outright by Postgres jsonb, so leaving this to the - // object case alone let the two backends disagree about the same value. + // Both of these run for every type, not just the object branch: a value the + // decoder silently rewrites is stored verbatim, and the two backends do not + // agree on what they will store. Postgres jsonb rejects U+FFFD-producing + // input outright while SQLite's json_valid accepts it, so the same request + // succeeds on one deployment and fails on the other. + // + // Raw bytes and escapes are separate paths to the same substitution: an + // invalid UTF-8 byte arrives in the body as itself, a lone surrogate as a + // \u escape, and encoding/json turns both into U+FFFD while reporting + // success. + if !utf8.Valid(trimmed) { + return errors.New("value is not valid UTF-8") + } if err := rejectLoneSurrogates(trimmed); err != nil { return err }