diff --git a/cspell.config.yml b/cspell.config.yml index 58f4192..d70769a 100644 --- a/cspell.config.yml +++ b/cspell.config.yml @@ -8,4 +8,5 @@ import: version: '0.2' words: - rcompare + - unstub - xsea diff --git a/packages/sea-builder/src/utils/downloadArchive.mts b/packages/sea-builder/src/utils/downloadArchive.mts index feee042..ba73a96 100644 --- a/packages/sea-builder/src/utils/downloadArchive.mts +++ b/packages/sea-builder/src/utils/downloadArchive.mts @@ -1,17 +1,131 @@ +import { createHash, randomUUID } from 'node:crypto'; import { createWriteStream } from 'node:fs'; +import { access, rename, rm } from 'node:fs/promises'; +import { basename } from 'node:path/posix'; +import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; +import type { ReadableStream as NodeWebReadableStream } from 'node:stream/web'; import type { DownloadFunction } from './types.mjs'; +/** Matches a 64-character lowercase or uppercase hex SHA-256 digest. */ +const SHA256_HEX = /^[0-9a-fA-F]{64}$/; + +/** + * Convert a Fetch API {@link Response.body} into a Node.js + * {@link Readable}. + * + * `fetch`'s `Response.body` is typed against the DOM `lib.dom.d.ts` + * `ReadableStream`, while {@link Readable.fromWeb} expects + * `node:stream/web`'s `ReadableStream`. The two describe the same + * runtime object but are not mutually assignable under TypeScript, so + * this is an explicit, narrowly-scoped cast rather than a bare + * `@ts-expect-error` on the call site. + * @param body A Fetch API response body stream. + * @returns The same stream, as a Node.js {@link Readable}. + */ +const toNodeReadable = (body: ReadableStream): Readable => + Readable.fromWeb(body as unknown as NodeWebReadableStream); + +/** + * Find the published SHA-256 checksum for a file in a `SHASUMS256.txt` + * listing. + * @param shasums Contents of the `SHASUMS256.txt` file. + * @param filename Archive file name to look up. + * @returns The expected lowercase hex SHA-256 digest. + * @throws {Error} If no parseable checksum line names `filename`, or the + * token found is not a 64-character hex digest. + */ +const findExpectedChecksum = (shasums: string, filename: string): string => { + const line = shasums + .split('\n') + .find((l) => l.trim().split(/\s+/).at(-1) === filename); + const checksum = line?.trim().split(/\s+/).at(0); + if (!checksum) { + throw new Error(`No checksum entry found for ${filename}`); + } + if (!SHA256_HEX.test(checksum)) { + throw new Error(`Malformed checksum entry for ${filename}: ${checksum}`); + } + return checksum.toLowerCase(); +}; + /** - * Download Node.js archive. + * Move a verified temporary download onto its final destination. + * + * `rename` is atomic on POSIX even when {@link dest} already exists, but + * Windows can refuse to replace a destination that another process holds + * open (`EPERM`/`EBUSY`) — notably when a concurrent {@link downloadArchive} + * call for the same target already won the race. In that case {@link dest} + * now holds bytes another call already checksum-verified, so this call's + * own (identical) temporary file is redundant and simply discarded rather + * than surfaced as an error. + * @param tempPath Path to the verified temporary file. + * @param dest Final destination path. + */ +const moveIntoPlace = async (tempPath: string, dest: string): Promise => { + try { + await rename(tempPath, dest); + } catch (error) { + await access(dest).catch(() => { + throw error; + }); + await rm(tempPath, { force: true }); + } +}; + +/** + * Download a Node.js archive to a temporary path, verify it against the + * published `SHASUMS256.txt` checksum, then atomically move it onto + * {@link dest}. A non-2xx response, a missing or unparseable checksum + * entry, or a checksum mismatch all throw and leave no file behind at + * {@link dest}; a failed or interrupted attempt removes its own + * temporary file so the next run does not treat it as a cache hit. * @param url Source URL. * @param dest Destination file path. */ export const downloadArchive: DownloadFunction = async (url, dest) => { const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to download ${url}: HTTP ${response.status}`); + } if (!response.body) { throw new Error('Response body is empty'); } - // @ts-expect-error - await pipeline(response.body, createWriteStream(dest)); + const filename = basename(new URL(url).pathname); + const shasumsUrl = new URL('SHASUMS256.txt', url); + const shasumsResponse = await fetch(shasumsUrl); + if (!shasumsResponse.ok) { + throw new Error( + `Failed to download ${shasumsUrl.toString()}: HTTP ${shasumsResponse.status}`, + ); + } + const expectedChecksum = findExpectedChecksum( + await shasumsResponse.text(), + filename, + ); + + const tempPath = `${dest}.download-${process.pid}-${randomUUID()}`; + try { + const hash = createHash('sha256'); + await pipeline( + toNodeReadable(response.body), + async function* (source) { + for await (const chunk of source) { + hash.update(chunk as Uint8Array); + yield chunk; + } + }, + createWriteStream(tempPath), + ); + const actualChecksum = hash.digest('hex'); + if (actualChecksum !== expectedChecksum) { + throw new Error( + `Checksum mismatch for ${filename}: expected ${expectedChecksum}, got ${actualChecksum}`, + ); + } + await moveIntoPlace(tempPath, dest); + } catch (error) { + await rm(tempPath, { force: true }); + throw error; + } }; diff --git a/packages/sea-builder/src/utils/downloadArchive.spec.mts b/packages/sea-builder/src/utils/downloadArchive.spec.mts new file mode 100644 index 0000000..af73365 --- /dev/null +++ b/packages/sea-builder/src/utils/downloadArchive.spec.mts @@ -0,0 +1,272 @@ +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { rename as nodeRename } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { downloadArchive } from './downloadArchive.mjs'; + +vi.mock('node:fs/promises', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, rename: vi.fn(actual.rename) }; +}); + +const ARCHIVE_CONTENT = 'fake-archive-bytes'; +const ARCHIVE_NAME = 'node-v22.23.1-linux-x64.tar.gz'; +const ARCHIVE_HASH = createHash('sha256').update(ARCHIVE_CONTENT).digest('hex'); +const ARCHIVE_URL = `https://nodejs.org/dist/v22.23.1/${ARCHIVE_NAME}`; + +const bodyStream = (content: string): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(content)); + controller.close(); + }, + }); + +const erroringBodyStream = ( + partial: string, + error: Error, +): ReadableStream => + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(partial)); + controller.error(error); + }, + }); + +const shasumsResponse = (body: string, ok = true, status = 200) => ({ + ok, + status, + text: async () => body, +}); + +const archiveResponse = ( + body: ReadableStream | null, + ok = true, + status = 200, +) => ({ body, ok, status }); + +let dir: string; +let dest: string; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'download-archive-')); + dest = join(dir, ARCHIVE_NAME); +}); + +afterEach(() => { + rmSync(dir, { force: true, recursive: true }); + vi.unstubAllGlobals(); +}); + +describe('downloadArchive', () => { + it('downloads and verifies a valid archive', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${ARCHIVE_HASH} ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await downloadArchive(ARCHIVE_URL, dest); + + expect(existsSync(dest)).toBe(true); + expect(readFileSync(dest, 'utf8')).toBe(ARCHIVE_CONTENT); + expect(readdirSync(dir)).toEqual([ARCHIVE_NAME]); + }); + + it('looks up the checksum by the URL filename, not the dest filename', async () => { + const renamedDest = join(dir, 'renamed-locally.tar.gz'); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${ARCHIVE_HASH} ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await downloadArchive(ARCHIVE_URL, renamedDest); + + expect(existsSync(renamedDest)).toBe(true); + expect(readFileSync(renamedDest, 'utf8')).toBe(ARCHIVE_CONTENT); + }); + + it('accepts an uppercase-hex checksum entry', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${ARCHIVE_HASH.toUpperCase()} ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await downloadArchive(ARCHIVE_URL, dest); + + expect(readFileSync(dest, 'utf8')).toBe(ARCHIVE_CONTENT); + }); + + it('throws on a checksum entry that is not a 64-character hex digest', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`not-a-hash ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /Malformed checksum entry/, + ); + expect(existsSync(dest)).toBe(false); + }); + + it('treats a rename refused by an already-placed destination as success', async () => { + writeFileSync(dest, ARCHIVE_CONTENT); + vi.mocked(nodeRename).mockRejectedValueOnce( + Object.assign(new Error('EPERM: operation not permitted, rename'), { + code: 'EPERM', + }), + ); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${ARCHIVE_HASH} ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).resolves.toBeUndefined(); + + expect(readFileSync(dest, 'utf8')).toBe(ARCHIVE_CONTENT); + expect(readdirSync(dir)).toEqual([ARCHIVE_NAME]); + }); + + it('propagates a rename failure when the destination was not placed by a winner', async () => { + const eperm = Object.assign( + new Error('EPERM: operation not permitted, rename'), + { code: 'EPERM' }, + ); + vi.mocked(nodeRename).mockRejectedValueOnce(eperm); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${ARCHIVE_HASH} ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow(eperm); + expect(existsSync(dest)).toBe(false); + expect(readdirSync(dir)).toEqual([]); + }); + + it('throws and leaves no file for a non-2xx archive response', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => archiveResponse(null, false, 404)), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /HTTP 404/, + ); + expect(existsSync(dest)).toBe(false); + }); + + it('throws and leaves no file for a non-2xx SHASUMS256.txt response', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse('', false, 404) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /HTTP 404/, + ); + expect(existsSync(dest)).toBe(false); + }); + + it('throws when no checksum entry names the archive', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse('deadbeef some-other-file.tar.gz\n') + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /No checksum entry/, + ); + expect(existsSync(dest)).toBe(false); + }); + + it('throws and leaves no file when the checksum does not match', async () => { + const wrongChecksum = 'd'.repeat(64); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${wrongChecksum} ${ARCHIVE_NAME}\n`) + : archiveResponse(bodyStream(ARCHIVE_CONTENT)), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /Checksum mismatch/, + ); + expect(existsSync(dest)).toBe(false); + }); + + it('leaves no temporary file when the download stream errors mid-transfer', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => + input.toString().endsWith('SHASUMS256.txt') + ? shasumsResponse(`${ARCHIVE_HASH} ${ARCHIVE_NAME}\n`) + : archiveResponse( + erroringBodyStream( + 'partial-bytes', + new Error('network interrupted'), + ), + ), + ), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /network interrupted/, + ); + expect(existsSync(dest)).toBe(false); + expect(readdirSync(dir)).toEqual([]); + }); + + it('throws when the archive response body is empty', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => archiveResponse(null)), + ); + + await expect(downloadArchive(ARCHIVE_URL, dest)).rejects.toThrow( + /Response body is empty/, + ); + expect(existsSync(dest)).toBe(false); + }); +}); diff --git a/packages/sea-builder/src/utils/toPipelineSource.mts b/packages/sea-builder/src/utils/toPipelineSource.mts deleted file mode 100644 index 29ef62e..0000000 --- a/packages/sea-builder/src/utils/toPipelineSource.mts +++ /dev/null @@ -1,16 +0,0 @@ -import type { PipelineSource, Stream } from 'node:stream'; - -/** - * Convert a stream to a pipeline source. - * @param s Stream to convert. - * @returns Pipeline source stream. - * @throws {TypeError} If the stream is not readable. - */ -export const toPipelineSource = (s: Stream): PipelineSource => { - if ((s as unknown as NodeJS.ReadableStream).readable) { - return s as unknown as NodeJS.ReadableStream; - } - const msg = - 'Streams that are not readable cannot be used as pipeline sources.'; - throw new TypeError(msg); -}; diff --git a/packages/sea-builder/src/utils/toPipelineSource.spec.mts b/packages/sea-builder/src/utils/toPipelineSource.spec.mts deleted file mode 100644 index 5ecebf8..0000000 --- a/packages/sea-builder/src/utils/toPipelineSource.spec.mts +++ /dev/null @@ -1,18 +0,0 @@ -import { PassThrough, Writable } from 'node:stream'; -import { describe, expect, it } from 'vitest'; -import { toPipelineSource } from './toPipelineSource.mjs'; - -describe('toPipelineSource', () => { - it('returns the stream when readable', () => { - const stream = new PassThrough(); - expect(toPipelineSource(stream)).toBe(stream); - }); - - it('throws for non readable streams', () => { - const stream = new Writable(); - Object.defineProperty(stream, 'readable', { value: false }); - expect(() => toPipelineSource(stream as unknown as PassThrough)).toThrow( - TypeError, - ); - }); -});