From bd6a89a43b82d44dea4bd736888a58ba3f53c8eb Mon Sep 17 00:00:00 2001 From: David Idol Date: Tue, 1 Sep 2020 23:07:33 -0700 Subject: [PATCH 1/3] Add bin to whitelist --- server/src/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/constants.ts b/server/src/constants.ts index f71dc3c..752960b 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -14,6 +14,7 @@ export const FILE_EXT_WHITELIST = [ '.jpg', '.jpeg', '.srt', + '.bin', ] export const FILE_TYPE_BLACKLIST = [ From 6f9dd7af83cc6929912a15ae583138b9c8cdbccc Mon Sep 17 00:00:00 2001 From: David Idol Date: Sun, 6 Sep 2020 22:37:15 -0700 Subject: [PATCH 2/3] Properly handle uploads without audio files --- .../ts/components/Beatmap/BeatmapResult.tsx | 50 ++++++------- client/src/ts/components/Beatmap/Detail.tsx | 50 ++++++------- .../src/ts/components/Beatmap/Statistics.tsx | 9 +++ client/src/ts/remote/beatmap.d.ts | 1 + client/src/ts/store/audio/actions.ts | 4 ++ server/src/constants.ts | 3 +- server/src/mongo/models/Beatmap.ts | 2 + server/src/routes/upload/errors.ts | 14 ++++ server/src/routes/upload/parseBeatmap.ts | 72 ++++++++++++++----- server/src/routes/upload/types.ts | 19 +++++ 10 files changed, 156 insertions(+), 68 deletions(-) diff --git a/client/src/ts/components/Beatmap/BeatmapResult.tsx b/client/src/ts/components/Beatmap/BeatmapResult.tsx index f2c57cf..76d4b22 100644 --- a/client/src/ts/components/Beatmap/BeatmapResult.tsx +++ b/client/src/ts/components/Beatmap/BeatmapResult.tsx @@ -113,30 +113,32 @@ const BeatmapResult: FunctionComponent = ({
- { - e.preventDefault() - - if (preview.state === 'playing' && preview.key === map.key) { - stopPreview() - } else { - previewBeatmap(map) - } - }} - > - {preview.key !== map.key - ? 'Preview' - : preview.loading - ? 'Preview' - : preview.error !== null - ? 'Playback error!' - : 'Stop Preview'} - + {!map.metadata.requiresExternalAudioFile && ( + { + e.preventDefault() + + if (preview.state === 'playing' && preview.key === map.key) { + stopPreview() + } else { + previewBeatmap(map) + } + }} + > + {preview.key !== map.key + ? 'Preview' + : preview.loading + ? 'Preview' + : preview.error !== null + ? 'Playback error!' + : 'Stop Preview'} + + )} OneClick™ = ({ Download OneClick™ Install - { - e.preventDefault() - - if (preview.state === 'playing' && preview.key === map.key) { - stopPreview() - } else { - previewBeatmap(map) - } - }} - > - {preview.loading - ? '.' - : preview.key !== map.key - ? 'Preview' - : preview.error !== null - ? 'Playback error!' - : 'Stop Preview'} - + {!map.metadata.requiresExternalAudioFile && ( + { + e.preventDefault() + + if (preview.state === 'playing' && preview.key === map.key) { + stopPreview() + } else { + previewBeatmap(map) + } + }} + > + {preview.loading + ? '.' + : preview.key !== map.key + ? 'Preview' + : preview.error !== null + ? 'Playback error!' + : 'Stop Preview'} + + )} {/* View on BeastSaber */} copyBSR(e)}> {copied ? ( diff --git a/client/src/ts/components/Beatmap/Statistics.tsx b/client/src/ts/components/Beatmap/Statistics.tsx index 5c55cce..27429c0 100644 --- a/client/src/ts/components/Beatmap/Statistics.tsx +++ b/client/src/ts/components/Beatmap/Statistics.tsx @@ -90,6 +90,15 @@ export const BeatmapStats: FunctionComponent = ({ hover='Beatmap Duration' /> ) : null} + + {isFullMap(map) && map.metadata.requiresExternalAudioFile ? ( + + ) : null} ) } diff --git a/client/src/ts/remote/beatmap.d.ts b/client/src/ts/remote/beatmap.d.ts index 667cc65..37d9231 100644 --- a/client/src/ts/remote/beatmap.d.ts +++ b/client/src/ts/remote/beatmap.d.ts @@ -16,6 +16,7 @@ declare interface IBeatmap { songSubName: string songAuthorName: string levelAuthorName: string + requiresExternalAudioFile: boolean bpm: number duration?: number diff --git a/client/src/ts/store/audio/actions.ts b/client/src/ts/store/audio/actions.ts index e12ae8a..2a0914a 100644 --- a/client/src/ts/store/audio/actions.ts +++ b/client/src/ts/store/audio/actions.ts @@ -10,6 +10,10 @@ export const previewBeatmap: ( ) => TypedThunk = beatmap => async (dispatch, getState) => { stopPreview()(dispatch, getState) + if (beatmap.metadata.requiresExternalAudioFile) { + return + } + dispatch({ payload: true, type: AudioActionTypes.SET_LOADING, diff --git a/server/src/constants.ts b/server/src/constants.ts index 752960b..bdc4bb7 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -43,7 +43,8 @@ export const FILE_TYPE_BLACKLIST = [ ] const SCHEMA_BASE_URI = - 'https://raw.githubusercontent.com/lolPants/beatmap-schemas/master/schemas' + 'https://raw.githubusercontent.com/idolize/beatmap-schemas/master/schemas' export const SCHEMA_INFO = `${SCHEMA_BASE_URI}/info.schema.json` export const SCHEMA_DIFFICULTY = `${SCHEMA_BASE_URI}/difficulty.schema.json` +export const SCHEMA_AUDIO_CONFIG = `${SCHEMA_BASE_URI}/audio.schema.json` diff --git a/server/src/mongo/models/Beatmap.ts b/server/src/mongo/models/Beatmap.ts index 9c815e4..407133b 100644 --- a/server/src/mongo/models/Beatmap.ts +++ b/server/src/mongo/models/Beatmap.ts @@ -25,6 +25,7 @@ export interface IBeatmapLean { songSubName: string songAuthorName: string levelAuthorName: string + requiresExternalAudioFile: boolean duration: number bpm: number @@ -112,6 +113,7 @@ const schema: Schema = new Schema({ type: String, }, songSubName: { type: String, maxlength: 255, es_indexed: false }, + requiresExternalAudioFile: { type: Boolean, default: false }, bpm: { type: Number, required: true }, diff --git a/server/src/routes/upload/errors.ts b/server/src/routes/upload/errors.ts index 543c6b3..83cf639 100644 --- a/server/src/routes/upload/errors.ts +++ b/server/src/routes/upload/errors.ts @@ -141,3 +141,17 @@ export const ERR_BEATMAP_PARSE_TIMEOUT = new CodedError( 'ERR_BEATMAP_PARSE_TIMEOUT', 408 ) + +export const ERR_BEATMAP_INVALID_AUDIO_CONFIG = new CodedError( + 'invalid audio.config file', + 0x30012, + 'ERR_BEATMAP_INVALID_AUDIO_CONFIG', + 400 +) + +export const ERR_BEATMAP_MISSING_FINGERPRINT = new CodedError( + 'beatmap missing fingerprint.bin file', + 0x30013, + 'ERR_BEATMAP_MISSING_FINGERPRINT', + 400 +) diff --git a/server/src/routes/upload/parseBeatmap.ts b/server/src/routes/upload/parseBeatmap.ts index 1f28d80..4c23a8f 100644 --- a/server/src/routes/upload/parseBeatmap.ts +++ b/server/src/routes/upload/parseBeatmap.ts @@ -9,6 +9,7 @@ import { withFile } from 'tmp-promise' import { FILE_EXT_WHITELIST, FILE_TYPE_BLACKLIST, + SCHEMA_AUDIO_CONFIG, SCHEMA_DIFFICULTY, SCHEMA_INFO, } from '~constants' @@ -31,6 +32,8 @@ import { ERR_BEATMAP_DIFF_NOT_FOUND, ERR_BEATMAP_INFO_INVALID, ERR_BEATMAP_INFO_NOT_FOUND, + ERR_BEATMAP_INVALID_AUDIO_CONFIG, + ERR_BEATMAP_MISSING_FINGERPRINT, } from './errors' export const parseBeatmap: ( @@ -78,32 +81,58 @@ export const parseBeatmap: ( if (size.width !== size.height) throw ERR_BEATMAP_COVER_NOT_SQUARE if (size.width < 256 || size.height < 256) throw ERR_BEATMAP_COVER_TOO_SMOL + let audio: Buffer | undefined + let requiresExternalAudioFile = false + let audioConfigJSON: IAudioConfig | undefined + const audioEntry = zip.file(infoJSON._songFilename) if (audioEntry === null) { + // Check if this song requires an external audio file + const audioConfigFile = zip.file('audio.json') + if (audioConfigFile !== null) { + const audioConfig = await audioConfigFile.async('text') + if (!validJSON(audioConfig)) throw ERR_BEATMAP_INVALID_AUDIO_CONFIG + const audioConfigJSON = JSON.parse(audioConfig) + const validateAudioConfig = await schemas.compile(SCHEMA_AUDIO_CONFIG) + const audioConfigValid = validateAudioConfig(audioConfigJSON) + if (audioConfigValid === false) { + parseValidationError('audio.json', validateAudioConfig.errors) + } + + const fingerprint = zip.file('fingerprint.bin') + if (fingerprint !== null) { + requiresExternalAudioFile = true + } else { + throw ERR_BEATMAP_MISSING_FINGERPRINT + } + } + throw ERR_BEATMAP_AUDIO_NOT_FOUND(infoJSON._songFilename) } - const audio = await audioEntry.async('nodebuffer') - const audioType = fileType(audio) - if ( - audioType === undefined || - (audioType.mime !== 'audio/ogg' && audioType.mime !== 'audio/wav') - ) { - throw ERR_BEATMAP_AUDIO_INVALID - } + if (!requiresExternalAudioFile) { + audio = await audioEntry.async('nodebuffer') + const audioType = fileType(audio) + if ( + audioType === undefined || + (audioType.mime !== 'audio/ogg' && audioType.mime !== 'audio/wav') + ) { + throw ERR_BEATMAP_AUDIO_INVALID + } - const { name, ext } = parse(infoJSON._songFilename) - if (ext === '.ogg') { - const eggName = `${name}.egg` + const { name, ext } = parse(infoJSON._songFilename) + if (ext === '.ogg') { + const eggName = `${name}.egg` - zip.remove(infoJSON._songFilename) - zip.file(eggName, audio) + zip.remove(infoJSON._songFilename) + zip.file(eggName, audio) - infoJSON._songFilename = `${name}.egg` - infoDAT = `${JSON.stringify(infoJSON, null, 2)}\n` + infoJSON._songFilename = `${name}.egg` + infoDAT = `${JSON.stringify(infoJSON, null, 2)}\n` - zip.remove(infoDATName) - zip.file(infoDATName, infoDAT) + zip.remove(infoDATName) + zip.file(infoDATName, infoDAT) + } } const difficulties = ([] as IDifficultyBeatmap[]).concat( @@ -201,7 +230,7 @@ export const parseBeatmap: ( ) const sha1 = hash.digest('hex') - const duration = await getAudioDuration(sha1, audio) + const duration = await getAudioDuration(sha1, audio, audioConfigJSON) const parsed: IParsedBeatmap = { hash: sha1, @@ -210,6 +239,7 @@ export const parseBeatmap: ( songAuthorName: infoJSON._songAuthorName, songName: infoJSON._songName, songSubName: infoJSON._songSubName, + requiresExternalAudioFile, bpm: infoJSON._beatsPerMinute, duration, @@ -271,8 +301,12 @@ const inspectFile = async (file: JSZip.JSZipObject) => { const getAudioDuration = async ( hash: string, - audio: Buffer + audio?: Buffer, + audioConfig?: IAudioConfig ): Promise => { + if (audioConfig && !audio) { + return (audioConfig.lengthMs || 0) / 1000.0 + } const args = [ '-v', 'error', diff --git a/server/src/routes/upload/types.ts b/server/src/routes/upload/types.ts index 6c3b871..529ed35 100644 --- a/server/src/routes/upload/types.ts +++ b/server/src/routes/upload/types.ts @@ -72,6 +72,7 @@ declare interface IParsedBeatmap { songSubName: string songAuthorName: string levelAuthorName: string + requiresExternalAudioFile: boolean duration: number bpm: number @@ -111,3 +112,21 @@ declare interface IParsedDifficulty { njs: number njsOffset: number } + +declare interface IAudioConfig { + schemaVersion: number + lengthMs: number + notes?: string + downloadUrls?: string[] + knownGoodHashes?: Array<{ type: string, hash: string }> + patches?: { + delayStartMs?: number + padEndMs?: number + trim?: { + startMs?: number + endMs: number + } + fadeIn?: { startMs: number, durationMs: number } + fadeOut?: { startMs: number, durationMs: number } + } +} From 7eb6e809f434c387d8729c6efb413da27500a1ca Mon Sep 17 00:00:00 2001 From: David Idol Date: Sun, 6 Sep 2020 22:44:42 -0700 Subject: [PATCH 3/3] Revert schema GitHub link --- server/src/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/constants.ts b/server/src/constants.ts index bdc4bb7..98dcfbd 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -43,7 +43,7 @@ export const FILE_TYPE_BLACKLIST = [ ] const SCHEMA_BASE_URI = - 'https://raw.githubusercontent.com/idolize/beatmap-schemas/master/schemas' + 'https://raw.githubusercontent.com/lolPants/beatmap-schemas/master/schemas' export const SCHEMA_INFO = `${SCHEMA_BASE_URI}/info.schema.json` export const SCHEMA_DIFFICULTY = `${SCHEMA_BASE_URI}/difficulty.schema.json`