diff --git a/server/formatter.js b/server/formatter.js index 1539f63c..fe176707 100644 --- a/server/formatter.js +++ b/server/formatter.js @@ -106,20 +106,21 @@ function normalizeHtml(html) { function formatCode(html) { // Expand code blocks html = html.replace(/```(.*?)```/ig, (match, content) => { - // strip interior

tags added by google - content = content.replace(/(?:<\/p>

|)/g, '\n').replace(/<\/?p>/g, '').trim() - // try to find language hint within text block - const [, lang] = content.match(/^(.+?)\n/) || [] - - if (lang && hljs.getLanguage(lang)) { - // if the language hint exists and contains a valid language, remove it from the code block - content = content.replace(`${lang}\n`, '') - - const textOnlyContent = cheerio.load(content).text() - const highlighted = hljs.highlight(lang, textOnlyContent, true) - return `

${formatCodeContent(highlighted.value)}
` - } - return `
${formatCodeContent(content)}
` + return formatCodeBlock(content) + }) + + // Preformat native code blocks + // Unnest native code block start and end markers + html = html.replace(/]*>(ເ[23];)<\/span>/ig, (match, marker) => { + return marker + }) + + // Expand native code blocks + // Google docs interleaves the end-of-code marker with the following tag. eg: + //

my code block

my heading

+ // Make sure we match and retain the following tag + html = html.replace(/]*>(.*?)<\/p>(<[^>]*>)/ig, (match, content, followingTag) => { + return `${formatCodeBlock(content)}${followingTag}` }) // Replace double backticks with , for supporting backticks in inline code blocks @@ -152,6 +153,23 @@ function formatCode(html) { return html } +function formatCodeBlock(content) { + // strip interior

tags added by google + content = content.replace(/(?:<\/p>]*>|)/g, '\n').replace(/<\/?p>/g, '').trim() + // try to find language hint within text block + const [, lang] = content.match(/^(.+?)\n/) || [] + + if (lang && hljs.getLanguage(lang)) { + // if the language hint exists and contains a valid language, remove it from the code block + content = content.replace(`${lang}\n`, '') + + const textOnlyContent = cheerio.load(content).text() + const highlighted = hljs.highlight(lang, textOnlyContent, true) + return `

${formatCodeContent(highlighted.value)}
` + } + return `
${formatCodeContent(content)}
` +} + function formatCodeContent(content) { content = content.replace(/[‘’]|ȁ[89];/g, "'").replace(/[“”]|ȁ[CD];/g, '"') // remove smart quotes content = content.replace(/`/g, '`') // remove internal cases of backticks diff --git a/test/fixtures/supportedFormats.nativeCode.html b/test/fixtures/supportedFormats.nativeCode.html new file mode 100644 index 00000000..7f4e8d80 --- /dev/null +++ b/test/fixtures/supportedFormats.nativeCode.html @@ -0,0 +1 @@ +

Intro sentence.

// This is a code block

// Here's another line

Middle sentence.

# A second code block

Outro sentence.

/* Another code block */

Heading following a code block

diff --git a/test/unit/htmlProcessing.test.js b/test/unit/htmlProcessing.test.js index 529c375e..f08355e3 100644 --- a/test/unit/htmlProcessing.test.js +++ b/test/unit/htmlProcessing.test.js @@ -5,6 +5,7 @@ const {assert} = require('chai') let {getProcessedDocAttributes} = require('../../server/formatter') const docPath = path.join(__dirname, '../fixtures/supportedFormats.html') +const docPathNativeCode = path.join(__dirname, '../fixtures/supportedFormats.nativeCode.html') // helper function to stub the doc and get a section of the returned document function stubbedProcessedDoc(unprocessedHtml, editorName) { @@ -13,15 +14,33 @@ function stubbedProcessedDoc(unprocessedHtml, editorName) { } describe('HTML processing', () => { - const testGlobal = { - rawHTML: null, - output: () => {}, - processedHTML: null - } + const testGlobal = {} + const condenseHtml = (html) => html.replace(/\n/g, '').replace(/>\s+<') + beforeAll(() => { - testGlobal.rawHTML = fs.readFileSync(docPath, {encoding: 'utf8'}) - testGlobal.processedHTML = stubbedProcessedDoc(testGlobal.rawHTML).html - testGlobal.output = cheerio.load(testGlobal.processedHTML) + // General supported formats + testGlobal.general = {} + testGlobal.general.rawHTML = fs.readFileSync(docPath, {encoding: 'utf8'}) + testGlobal.general.processedHTML = stubbedProcessedDoc(testGlobal.general.rawHTML).html + testGlobal.general.output = cheerio.load(testGlobal.general.processedHTML) + + // Native code + testGlobal.native = {} + testGlobal.native.rawHTML = fs.readFileSync(docPathNativeCode, {encoding: 'utf8'}) + testGlobal.native.processedHTML = stubbedProcessedDoc(testGlobal.native.rawHTML).html + testGlobal.native.output = cheerio.load(testGlobal.native.processedHTML) + + // Supported formats with inline code enabled + jest.resetModules() + process.env.ALLOW_INLINE_CODE = 'true' + // remove formatter from require cache to recognize changed env variable + delete require.cache[require.resolve('../../server/formatter')] + getProcessedDocAttributes = require('../../server/formatter').getProcessedDocAttributes + + testGlobal.inlineCode = {} + testGlobal.inlineCode.rawHTML = fs.readFileSync(docPath, {encoding: 'utf8'}) + testGlobal.inlineCode.processedHTML = stubbedProcessedDoc(testGlobal.inlineCode.rawHTML).html + testGlobal.inlineCode.output = cheerio.load(testGlobal.inlineCode.processedHTML) }) it('does not throw when revision data is unavailable', () => { @@ -30,38 +49,38 @@ describe('HTML processing', () => { }) it('strips unnecessary styles', () => { - const header = testGlobal.output('h2') + const header = testGlobal.general.output('h2') assert.equal(null, header.attr('style')) }) it('strips unnecessary  s', () => { - const introHTML = testGlobal.output("p:contains('Basic text format')").html() + const introHTML = testGlobal.general.output("p:contains('Basic text format')").html() assert.match(introHTML, /Text color and highlighting/) }) describe('inline formats', () => { it('preserves bolds', () => { - const boldSpan = testGlobal.output("span:contains('bold')").first() + const boldSpan = testGlobal.general.output("span:contains('bold')").first() assert.equal('font-weight:700', boldSpan.attr('style')) }) it('preserves italics', () => { - const italicSpan = testGlobal.output("span:contains('italic')").first() + const italicSpan = testGlobal.general.output("span:contains('italic')").first() assert.equal('font-style:italic', italicSpan.attr('style')) }) it('preserves underlines', () => { - const underlinedSpan = testGlobal.output("span:contains('underline')").first() + const underlinedSpan = testGlobal.general.output("span:contains('underline')").first() assert.equal('text-decoration:underline', underlinedSpan.attr('style')) }) it('preserves combined formats', () => { - const combinedSpan = testGlobal.output("span:contains('combined')").first() + const combinedSpan = testGlobal.general.output("span:contains('combined')").first() assert.equal('font-style:italic;font-weight:700;text-decoration:underline', combinedSpan.attr('style')) }) it('preserves image widths', () => { - const imageWidth = testGlobal.output('img').first() + const imageWidth = testGlobal.general.output('img').first() const widthMatch = imageWidth.attr('style').match('width') assert.isNotNull(widthMatch) }) @@ -69,71 +88,114 @@ describe('HTML processing', () => { describe('list handling', () => { it('preserves classing on lists', () => { - const ol = testGlobal.output('ol').first() + const ol = testGlobal.general.output('ol').first() assert.match(ol.attr('class'), /lst-/) }) it('presrves the associated style block for lists', () => { - const olClass = testGlobal.output('ol').first().attr('class').split(' ')[0] - assert.match(testGlobal.processedHTML, new RegExp(`ol.${olClass} {`)) + const olClass = testGlobal.general.output('ol').first().attr('class').split(' ')[0] + assert.match(testGlobal.general.processedHTML, new RegExp(`ol.${olClass} {`)) }) it('applies a level- class on lists to support indentation', () => { - const topLevelList = testGlobal.output("ul:contains('Item 1')").first() + const topLevelList = testGlobal.general.output("ul:contains('Item 1')").first() assert.match(topLevelList.attr('class'), / level-0/) - const nestedList = testGlobal.output("ul:contains('Item 1.1')").first() + const nestedList = testGlobal.general.output("ul:contains('Item 1.1')").first() assert.match(nestedList.attr('class'), / level-1/) }) }) describe('code block handling', () => { it('highlights registered languages', () => { - const codeBlock = testGlobal.output('pre > code[data-lang="javascript"]') + const codeBlock = testGlobal.general.output('pre > code[data-lang="javascript"]') assert.exists(codeBlock.html()) }) it('allows   as part of a code block', () => { - const codeBlock = testGlobal.output('pre > code[data-lang="javascript"]') + const codeBlock = testGlobal.general.output('pre > code[data-lang="javascript"]') assert.match(codeBlock.html(), /&nbsp/) }) it('preserves whitespace at the start of a line', () => { - const codeBlock = testGlobal.output('pre > code[data-lang="javascript"]') + const codeBlock = testGlobal.general.output('pre > code[data-lang="javascript"]') assert.match(codeBlock.html(), / +jQuery.fn.calcSubWidth/) }) it('scrubs smart quotes', () => { - const codeBlock = testGlobal.output('pre > code[data-lang="javascript"]') + const codeBlock = testGlobal.general.output('pre > code[data-lang="javascript"]') assert.match(codeBlock.html(), /singleQuotedStr = .*'str'/) assert.match(codeBlock.html(), /doubleQuotedStr = .*"str"/) }) it('allows unregistered languages', () => { - const codeBlock = testGlobal.output('pre') + const codeBlock = testGlobal.general.output('pre') assert.match(codeBlock.html(), /1 \+ 1 == 5/) }) it('retains code block backticks', () => { - const codeBlock = testGlobal.output('pre > code[data-lang="javascript"]') + const codeBlock = testGlobal.general.output('pre > code[data-lang="javascript"]') assert.match(codeBlock.html(), /`/) }) it('retains inline code backticks', () => { - const codeBlock = testGlobal.output("code:contains('backtick')") + const codeBlock = testGlobal.general.output("code:contains('backtick')") assert.match(codeBlock.html(), /`backtick`/) }) }) + describe('native code block handling', () => { + it('formats the code blocks', () => { + const codeBlock = testGlobal.native.output('pre > code') + assert.exists(codeBlock.html()) + assert.equal(codeBlock.length, 3) + }) + + it('leaves the trailing heading intact', () => { + const heading = testGlobal.native.output('h2') + assert.equal(heading.html(), 'Heading following a code block') + }) + + it('removes code block marker unicode characters', () => { + assert.notInclude(testGlobal.native.processedHTML, '') + assert.notInclude(testGlobal.native.processedHTML, '') + }) + + it('unnests start and end markers', () => { + const html = condenseHtml(` +

+ \uEC03 + my code +

+

+ \uEC02 +

+ `) + const processedHtml = stubbedProcessedDoc(html).html + assert.equal(processedHtml, '
my code
\n

') + }) + + it('removes interior

tags with attributes', () => { + const html = condenseHtml(` +

\uEC03my code

+

more code

+

\uEC02

+ `) + console.log(html) + const processedHtml = stubbedProcessedDoc(html).html + assert.equal(processedHtml, '
my code\nmore code
\n

') + }) + }) + describe('inline code handling', () => { describe('with inline code disabled', () => { it('does not modify code block content', () => { - const codeBlock = testGlobal.output("pre:contains('codeblocks will not')") + const codeBlock = testGlobal.general.output("pre:contains('codeblocks will not')") assert.match(codeBlock.html(), /<.*%-.*%>/) }) it('does not unescape delimited code', () => { - const className = testGlobal.output("p:contains('.purplePapyrus')") + const className = testGlobal.general.output("p:contains('.purplePapyrus')") const styleTag = className.prev() const openingTag = styleTag.prev() @@ -143,25 +205,14 @@ describe('HTML processing', () => { }) describe('with inline code enabled', () => { - beforeAll(() => { - jest.resetModules() - process.env.ALLOW_INLINE_CODE = 'true' - // remove formatter from require cache to recognize changed env variable - delete require.cache[require.resolve('../../server/formatter')] - getProcessedDocAttributes = require('../../server/formatter').getProcessedDocAttributes - const rawHTML = fs.readFileSync(docPath, {encoding: 'utf8'}) - const processedHTML = stubbedProcessedDoc(rawHTML).html - testGlobal.codeEnabledOut = cheerio.load(processedHTML) - }) - it('does not modify code block content', () => { - const codeBlock = testGlobal.codeEnabledOut("pre:contains('codeblocks will not')") + const codeBlock = testGlobal.inlineCode.output("pre:contains('codeblocks will not')") assert.match(codeBlock.html(), /<.*%-.*%>/) }) it('properly unescapes delimited code', () => { - const style = testGlobal.codeEnabledOut("style:contains('.purplePapyrus')") - const styledDiv = testGlobal.codeEnabledOut('div.purplePapyrus') + const style = testGlobal.inlineCode.output("style:contains('.purplePapyrus')") + const styledDiv = testGlobal.inlineCode.output('div.purplePapyrus') assert.exists(style) assert.exists(styledDiv) @@ -173,11 +224,11 @@ describe('HTML processing', () => { describe('comment handling', () => { it('strips comments', () => { - assert.notMatch(testGlobal.processedHTML, /This comment text will not appear/) + assert.notMatch(testGlobal.general.processedHTML, /This comment text will not appear/) }) it('strips inline comment anchors', () => { - const commentAnchorParent = testGlobal.output("p:contains('will be stripped from the')") + const commentAnchorParent = testGlobal.general.output("p:contains('will be stripped from the')") assert.notMatch(commentAnchorParent, /\[a\]/) }) })