fix(encoding): set content encoding to UTF-8

content/router.xql

@@ -465,15 +465,17 @@ declare %private function router:write-response ($default-code as xs:integer, $r
         router:get-content-type-for-code($config, $code, "application/xml")
     ))
 
+    let $method := router:method-for-content-type($content-type)
+
     return (
         response:set-status-code($code),
         router:set-additional-headers($response?($router:RESPONSE_HEADERS)),
         if ($code = 204) then 
             ()
         else 
             (
-                response:set-header("Content-Type", $content-type),
-                util:declare-option("output:method", router:method-for-content-type($content-type)),
+                response:set-header("Content-Type", router:content-type-header($content-type, $method)),
+                util:declare-option("output:method", $method),
                 $response?($router:RESPONSE_BODY)
             )
     )
@@ -497,6 +499,38 @@ declare %private function router:safe-set-header ($header as xs:string, $value a
     else response:set-header($header, $value)
 };
 
+(:~
+ : Build the actual Content-Type header value for a response.
+ :
+ : eXist-db's serializer always writes text-based output as UTF-8 unless a
+ : route explicitly overrides it - there is no per-route encoding to track,
+ : so a response's charset isn't something routes.json should have to
+ : declare. Derive it from the already-computed serialization $method
+ : instead, for every text-ish type, so it's never missing or hand-wired
+ : (a missing charset leaves the client to guess the encoding, which is how
+ : non-Latin scripts end up mangled into mojibake).
+ : A type that already carries parameters (the operation explicitly asked
+ : for something specific) is left untouched. application/json is left
+ : alone too - RFC 8259 says senders shouldn't add a charset parameter to
+ : it, since JSON text is always UTF-8 by definition.
+ :
+ : $method "text" is ambiguous: router:method-for-content-type also returns it
+ : as a passthrough for binary types (e.g. application/octet-stream, image/png)
+ : that aren't otherwise recognized, so it can't be trusted alone - a charset
+ : is only added for it when $type itself is actually a text/* media type.
+ :)
+declare %private function router:content-type-header ($type as xs:string, $method as xs:string) as xs:string {
+    if (contains($type, ";")) then
+        $type
+    else if (
+        $method = ("html5", "xhtml", "xml") or
+        ($method = "text" and starts-with($type, "text/"))
+    ) then
+        $type || "; charset=UTF-8"
+    else
+        $type
+};
+
 (:~
  : Q: binary types?
  : XSLT default values: "xml", "xhtml", "html", "text", "json", "adaptive"

test/app/api.json

@@ -1035,6 +1035,31 @@
                 }
             }
         },
+        "/api/encoding-test": {
+            "get": {
+                "summary": "Echo an arbitrary Content-Type back through roaster:response",
+                "description": "Used to test how the router derives the actual Content-Type response header (e.g. charset handling) for a given media type.",
+                "operationId": "api:encoding-test",
+                "tags": [
+                    "query"
+                ],
+                "parameters": [
+                    {
+                        "name": "type",
+                        "in": "query",
+                        "required": false,
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Body sent with the requested Content-Type"
+                    }
+                }
+            }
+        },
         "/api/errors/handle": {
             "get": {
                 "summary": "Test error handler",

test/app/modules/api.xql

@@ -115,6 +115,10 @@ declare function api:avatar ($request as map(*)) {
     </svg>
 };
 
+declare function api:encoding-test ($request as map(*)) {
+    roaster:response(200, ($request?parameters?type, 'text/html')[1] , <html><body><h1>café 💩</h1></body></html>)
+};
+
 (:~
  : A route handler that returns all parsed parameter values
  :)

test/charset.test.js

@@ -0,0 +1,66 @@
+const util = require('./util.js')
+const chai = require('chai')
+const expect = chai.expect
+
+// Regression test for missing response charsets.
+describe('Content-Type header charset handling', function () {
+    const phrase = 'café 💩'
+
+    function fetchAs (type) {
+        return util.axios.get('api/encoding-test', {
+            params: { type },
+            responseType: 'arraybuffer'
+        })
+    }
+
+    describe('text-ish types get an explicit UTF-8 charset', function () {
+        const cases = [
+            ['application/xml', 'application/xml; charset=UTF-8'],
+            ['application/xhtml+xml', 'application/xhtml+xml; charset=UTF-8'],
+            ['image/svg+xml', 'image/svg+xml; charset=UTF-8'],
+            // eXist-db's servlet recognizes text/* as textual and re-normalizes
+            // the header when it sets the response's character encoding -
+            // still charset=utf-8, just reformatted (no space, lowercase).
+            ['text/html', 'text/html;charset=utf-8'],
+            ['text/plain', 'text/plain;charset=utf-8']
+        ]
+
+        cases.forEach(([type, expectedHeader]) => {
+            it(`${type} -> ${expectedHeader}`, async function () {
+                const res = await fetchAs(type)
+                expect(res.headers['content-type']).to.equal(expectedHeader)
+                expect(Buffer.from(res.data).toString('utf-8')).to.include(phrase)
+            })
+        })
+    })
+
+    describe('types that must not get a charset appended', function () {
+        // application/json is handled separately: it correctly serializes the
+        // string as a quoted JSON value, so its bytes aren't the raw phrase.
+        it('application/json is returned without a charset parameter', async function () {
+            const res = await fetchAs('application/json')
+            expect(res.headers['content-type']).to.equal('application/json')
+            expect(Buffer.from(res.data).toString('utf-8')).to.include(phrase)
+        })
+
+        const cases = [
+            'application/octet-stream', // binary passthrough via the "text" method
+            'image/png', // binary passthrough via the "text" method
+            'audio/mpeg' // binary passthrough via the "text" method
+        ]
+
+        cases.forEach((type) => {
+            it(`${type} is returned without a charset parameter`, async function () {
+                const res = await fetchAs(type)
+                expect(res.headers['content-type']).to.equal(type)
+                expect(Buffer.from(res.data).toString('utf-8')).to.include(phrase)
+            })
+        })
+    })
+
+    it('leaves a type that already declares its own parameters untouched', async function () {
+        const res = await fetchAs('application/xml; charset=us-ascii')
+        expect(res.headers['content-type']).to.equal('application/xml; charset=us-ascii')
+        expect(Buffer.from(res.data).toString('utf-8')).to.include('café 💩')
+    })
+})

test/error.test.js

@@ -25,7 +25,8 @@ describe('Error reporting', function() {
         return util.axios.delete('api/errors')
             .catch(function(error) {
                 expect(error.response.status).to.equal(403)
-                expect(error.response.headers['content-type']).to.equal('application/xml')
+                // xml responses carry an explicit UTF-8 charset (see charset.test.js)
+                expect(error.response.headers['content-type']).to.equal('application/xml; charset=UTF-8')
                 expect(error.response.data).to.equal('<forbidden/>')
             })
     })
@@ -34,7 +35,10 @@ describe('Error reporting', function() {
         return util.axios.get('api/errors/handle')
             .catch(function(error) {
                 expect(error.response.status).to.equal(500)
-                expect(error.response.headers['content-type']).to.equal('text/html')
+                // html responses carry an explicit UTF-8 charset too, though
+                // eXist-db's servlet re-normalizes the header it is given
+                // (no space, lowercase) when it sets the character encoding
+                expect(error.response.headers['content-type']).to.equal('text/html;charset=utf-8')
                 expect(error.response.data).to.contain('$undefined')
             })
     })

test/mediatype.test.js

@@ -753,7 +753,9 @@ describe("Retrieving an SVG image", function () {
         expect(response.status).to.equal(200)
     })
     it("was sent with the correct Content-Type header", function () {
-        expect(response.headers['content-type']).to.equal('image/svg+xml')
+        // image/svg+xml is XML-flavored text, so it gets an explicit UTF-8
+        // charset just like application/xml (see charset.test.js).
+        expect(response.headers['content-type']).to.equal('image/svg+xml; charset=UTF-8')
     })
     it("is pretty printed", function () {
         expect(response.data).to.equal(avatarImage)