data: add September event song charts

scripts/download-sus-charts.mjs

@@ -0,0 +1,121 @@
+// Download and decrypt selected Hololive Dreams SUS resources from Octo CDN.
+// Kept dependency-free so chart collection does not depend on a local Python runtime.
+import { createDecipheriv, createHash } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const OCTO_API_KEY = 'B46OtKlGGHoz6sxbOWDe3VUvBsagXxr5av38IQIKUKo=';
+const OCTO_CLIENT_KEY = 'CwFhQ+S5m4nERWVaq5oFZIP0cZLc0j7O/zllG0UYVNo=';
+const OCTO_LIST_URL = 'https://us.game-hololive-dreams.com/asset/v2/pub/a/5/v/200001/list/0';
+const resourceMagic = Buffer.from('QUAVMAGIC');
+
+function readVarint(bytes, offset) {
+  let value = 0;
+  let shift = 0;
+  while (offset < bytes.length) {
+    const current = bytes[offset++];
+    value += (current & 0x7f) * (2 ** shift);
+    if ((current & 0x80) === 0) return [value, offset];
+    shift += 7;
+  }
+  throw new Error('잘린 Octo protobuf입니다.');
+}
+
+function protobufFields(bytes) {
+  const fields = [];
+  for (let offset = 0; offset < bytes.length;) {
+    const [tag, afterTag] = readVarint(bytes, offset);
+    offset = afterTag;
+    const field = tag >> 3;
+    const wireType = tag & 7;
+    let value;
+    if (wireType === 0) [value, offset] = readVarint(bytes, offset);
+    else if (wireType === 2) {
+      let length;
+      [length, offset] = readVarint(bytes, offset);
+      value = bytes.subarray(offset, offset + length);
+      offset += length;
+    } else if (wireType === 1) {
+      value = bytes.subarray(offset, offset + 8);
+      offset += 8;
+    } else if (wireType === 5) {
+      value = bytes.subarray(offset, offset + 4);
+      offset += 4;
+    } else throw new Error(`지원하지 않는 protobuf wire type: ${wireType}`);
+    fields.push([field, value]);
+  }
+  return fields;
+}
+
+function decodeEntry(bytes) {
+  const entry = {};
+  for (const [field, value] of protobufFields(bytes)) {
+    if (field === 2) entry.name = Buffer.from(value).toString('utf8');
+    else if (field === 7) entry.objectName = Buffer.from(value).toString('utf8');
+  }
+  return entry;
+}
+
+function decryptCatalog(encrypted) {
+  const key = createHash('sha256').update(OCTO_API_KEY).digest();
+  const decipher = createDecipheriv('aes-256-cbc', key, encrypted.subarray(0, 16));
+  const plaintext = Buffer.concat([decipher.update(encrypted.subarray(16)), decipher.final()]);
+  const entries = [];
+  let urlFormat = '';
+  let revisionId = null;
+  for (const [field, value] of protobufFields(plaintext)) {
+    if (field === 1) revisionId = value;
+    else if (field === 3) entries.push(decodeEntry(value));
+    else if (field === 4) urlFormat = Buffer.from(value).toString('utf8');
+  }
+  if (!urlFormat || revisionId === null) throw new Error('Octo 카탈로그 형식이 바뀌었습니다.');
+  return { revisionId, urlFormat, entries };
+}
+
+function resourceMask(name) {
+  // Exact equivalent of holodori-asset-tools/crypto/mask.py.
+  const mask = Buffer.alloc(name.length * 2);
+  for (let index = 0; index < name.length; index += 1) {
+    const character = name.charCodeAt(index) & 0xff;
+    mask[index * 2] = character;
+    mask[mask.length - 1 - index * 2] = (~character) & 0xff;
+  }
+  let hash = 0x7c;
+  for (const byte of mask) hash = ((((hash & 1) << 7) | (hash >> 1)) ^ byte) & 0xff;
+  return Buffer.from(mask.map((byte) => byte ^ hash));
+}
+
+function decryptResource(bytes, name) {
+  if (!bytes.subarray(0, resourceMagic.length).equals(resourceMagic)) return bytes;
+  const mask = resourceMask(name);
+  const output = Buffer.from(bytes.subarray(resourceMagic.length));
+  for (let index = 0; index < Math.min(256, output.length); index += 1) output[index] ^= mask[index % mask.length];
+  return output;
+}
+
+const requested = process.argv.slice(2);
+if (requested.length === 0) throw new Error('SUS 파일명 또는 곡 ID를 지정하세요. 예: m0344 m0348 m0527');
+const requestedNames = new Set(requested.flatMap((value) =>
+  /^m\d{4}$/.test(value) ? ['easy', 'normal', 'hard', 'expert'].map((difficulty) => `chart_${value}_${difficulty}.sus`) : [value]
+));
+
+const catalogResponse = await fetch(OCTO_LIST_URL, { headers: { 'X-OCTO-KEY': OCTO_API_KEY, 'X-APP-OCTO-KEY': OCTO_CLIENT_KEY } });
+if (!catalogResponse.ok) throw new Error(`Octo 카탈로그 요청 실패: ${catalogResponse.status}`);
+const catalog = decryptCatalog(Buffer.from(await catalogResponse.arrayBuffer()));
+const selected = catalog.entries.filter((entry) => requestedNames.has(entry.name));
+if (selected.length !== requestedNames.size) {
+  const found = new Set(selected.map((entry) => entry.name));
+  throw new Error(`카탈로그에 없는 SUS: ${[...requestedNames].filter((name) => !found.has(name)).join(', ')}`);
+}
+
+const outputDirectory = path.resolve('data/raw/sus');
+await mkdir(outputDirectory, { recursive: true });
+for (const entry of selected.sort((left, right) => left.name.localeCompare(right.name))) {
+  const response = await fetch(catalog.urlFormat.replace('{o}', entry.objectName));
+  if (!response.ok) throw new Error(`${entry.name} 다운로드 실패: ${response.status}`);
+  const plaintext = decryptResource(Buffer.from(await response.arrayBuffer()), entry.name);
+  if (!plaintext.toString('utf8', 0, 9).startsWith('#CREATE_')) throw new Error(`${entry.name} 복호화 결과가 SUS가 아닙니다.`);
+  await writeFile(path.join(outputDirectory, entry.name), plaintext);
+  console.log(`${entry.name} ${createHash('sha256').update(plaintext).digest('hex')}`);
+}
+console.log(`Octo revision ${catalog.revisionId}: ${selected.length}개 SUS 저장 완료`);

scripts/song-score-formulas.mjs

@@ -4,7 +4,9 @@ import { scoreRatioFromFormula } from '../src/lib/engine/score.ts';
 // Pinned research snapshot, with source URL, retrieval time and SHA-256.
 const reference = JSON.parse(await readFile(new URL('../docs/research/song-score-formula-reference.json', import.meta.url), 'utf8'));
 const susReference = JSON.parse(await readFile(new URL('../docs/research/sus-score-formulas.json', import.meta.url), 'utf8'));
-const rows = [...reference.charts, ...susReference.charts];
+// Newly recovered charts carry their full reproducibility record in validationCharts until
+// the next research snapshot is consolidated.
+const rows = [...reference.charts, ...susReference.charts, ...susReference.validationCharts.filter((row) => row.fullComboNoteCount)];
 const formulas = new Map(rows.map((row) => [`${row.songId}.${row.difficulty}`, row]));
 if (formulas.size !== rows.length) throw new Error('중복 악곡 점수 공식입니다.');
 

scripts/sus-to-exact-chart.mjs

@@ -0,0 +1,218 @@
+// Convert Hololive Dreams' decrypted SUS dialect to this application's compact score timeline.
+import { readFile, writeFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { susScoreWeight } from './sus-score-weight.mjs';
+import { formulaScoreMetadata } from './song-score-formulas.mjs';
+
+const alphabet = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+const difficulties = ['easy', 'normal', 'hard', 'expert'];
+
+function charValue(character) {
+  const value = alphabet.indexOf(character);
+  if (value < 0) throw new Error(`잘못된 SUS base62 값: ${character}`);
+  return value;
+}
+
+function parseCells(data, includeZero = false) {
+  const cells = [];
+  let position = 0;
+  for (let index = 0; index < data.length;) {
+    if (data.startsWith('0x', index)) {
+      position += charValue(data[index + 2]) * 62 + charValue(data[index + 3]);
+      index += 4;
+    } else {
+      const cell = data.slice(index, index + 2);
+      if (cell.length !== 2) throw new Error('잘린 SUS 셀입니다.');
+      if (includeZero ? cell !== '00' : cell[0] !== '0') cells.push({ position, cell });
+      position += 1;
+      index += 2;
+    }
+  }
+  return { cells, total: position };
+}
+
+function parseHeaders(text) {
+  const headers = new Map();
+  for (const raw of text.split(/\r?\n/)) {
+    const match = raw.match(/^#([A-Z_]+)\s+(.+)$/);
+    if (match) headers.set(match[1], match[2].trim().replaceAll('"', ''));
+  }
+  return headers;
+}
+
+function measureStarts(lengths) {
+  const starts = [];
+  let beat = 0;
+  for (let measure = 0; measure < 1000; measure += 1) {
+    starts[measure] = beat;
+    beat += lengths.get(measure) ?? 4;
+  }
+  return starts;
+}
+
+function parseSusTimeline(text) {
+  const headers = parseHeaders(text);
+  const lengths = new Map();
+  const bpmDefinitions = new Map();
+  const lines = text.split(/\r?\n/).map((raw) => raw.trim());
+  for (const line of lines) {
+    const bar = line.match(/^#(\d{3})02:(.+)$/);
+    if (bar) lengths.set(Number(bar[1]), Number(bar[2]));
+    const bpm = line.match(/^#BPM(..):(.+)$/);
+    if (bpm) bpmDefinitions.set(bpm[1], Number(bpm[2]));
+  }
+  const starts = measureStarts(lengths);
+  const toBeat = (measure, position, total) => starts[measure] + (lengths.get(measure) ?? 4) * position / total;
+  const bpmEvents = [];
+  const taps = [];
+  const flicks = [];
+  const holdStreams = new Map();
+  const visibleRelays = [];
+  const specialStarts = [];
+  for (const line of lines) {
+    const holdMatch = line.match(/^#(\d{3})3([0-9a-z])([0-9a-zA-Z]):(.+)$/);
+    if (holdMatch) {
+      const [, measureText, lane, channel, data] = holdMatch;
+      const measure = Number(measureText);
+      const parsed = parseCells(data);
+      const notes = parsed.cells.map(({ position, cell }) => ({ beat: toBeat(measure, position, parsed.total), lane: charValue(lane), kind: charValue(cell[0]), width: charValue(cell[1]) }));
+      visibleRelays.push(...notes.filter((note) => note.kind === 3));
+      const stream = charValue(channel);
+      if (!holdStreams.has(stream)) holdStreams.set(stream, []);
+      holdStreams.get(stream).push(...notes);
+      continue;
+    }
+    const match = line.match(/^#(\d{3})([0-9a-z])([0-9a-zA-Z]):(.+)$/);
+    if (!match) continue;
+    const [, measureText, family, lane, data] = match;
+    const measure = Number(measureText);
+    if (`${family}${lane}` === '08') {
+      for (const { position, cell } of parseCells(data, true).cells) bpmEvents.push({ beat: toBeat(measure, position, parseCells(data, true).total), bpm: bpmDefinitions.get(cell) ?? Number(headers.get('BASEBPM') ?? 120) });
+      continue;
+    }
+    if (`${family}${lane}` === '0B') {
+      for (const { position } of parseCells(data, true).cells) specialStarts.push(toBeat(measure, position, parseCells(data, true).total));
+      continue;
+    }
+    if (!['1', '5'].includes(family)) continue;
+    const parsed = parseCells(data);
+    const notes = parsed.cells.map(({ position, cell }) => ({ beat: toBeat(measure, position, parsed.total), lane: charValue(lane), kind: charValue(cell[0]), width: charValue(cell[1]) }));
+    if (family === '1') taps.push(...notes);
+    else if (family === '5') flicks.push(...notes);
+  }
+  if (bpmEvents.length === 0) bpmEvents.push({ beat: 0, bpm: Number(headers.get('BASEBPM') ?? 120) });
+  bpmEvents.sort((left, right) => left.beat - right.beat);
+  const uniqueBpmEvents = bpmEvents.filter((event, index) => index === 0 || event.beat !== bpmEvents[index - 1].beat);
+  function secondsAt(beat) {
+    let seconds = 0;
+    let previous = uniqueBpmEvents[0];
+    if (previous.beat > 0) seconds += previous.beat * 60 / Number(headers.get('BASEBPM') ?? 120);
+    for (let index = 1; index < uniqueBpmEvents.length && uniqueBpmEvents[index].beat <= beat; index += 1) {
+      const event = uniqueBpmEvents[index];
+      seconds += (event.beat - previous.beat) * 60 / previous.bpm;
+      previous = event;
+    }
+    seconds += (beat - previous.beat) * 60 / previous.bpm;
+    return seconds + Number(headers.get('WAVEOFFSET') ?? 0);
+  }
+  const key = (note) => `${note.beat}|${note.lane}`;
+  const tapKeys = new Set(taps.filter((note) => note.kind === 1 || note.kind === 2).map(key));
+  const standaloneFlickKeys = new Set(flicks.filter((note) => [1, 3, 4].includes(note.kind)).map(key));
+  const holds = [];
+  for (const stream of holdStreams.values()) {
+    const deduped = [...new Map(stream.sort((left, right) => left.beat - right.beat).map((note) => [`${note.beat}|${note.lane}|${note.kind}|${note.width}`, note])).values()];
+    const open = [];
+    const relays = [];
+    for (const note of deduped) {
+      if (note.kind === 1) open.push(note);
+      else if (note.kind === 2) {
+        const index = open.findIndex((start) => start.beat <= note.beat);
+        if (index < 0) continue;
+        const start = open.splice(index, 1)[0];
+        const inner = relays.filter((relay) => start.beat <= relay.beat && relay.beat <= note.beat);
+        for (const relay of inner) relays.splice(relays.indexOf(relay), 1);
+        holds.push([start, ...inner.sort((left, right) => left.beat - right.beat), note]);
+      } else relays.push(note);
+    }
+  }
+  const holdStartKeys = new Set(holds.map((hold) => key(hold[0])));
+  const holdEndKeys = new Set(holds.map((hold) => key(hold.at(-1))));
+  const notes = [];
+  // A co-located directional is one flick. A hold end directional is part of the hold and scores at normal weight.
+  for (const tap of taps) {
+    if (![1, 2].includes(tap.kind) || holdStartKeys.has(key(tap))) continue;
+    notes.push({ timeSeconds: secondsAt(tap.beat), type: 'regular', isFlick: standaloneFlickKeys.has(key(tap)) });
+  }
+  for (const flick of flicks) {
+    if (![1, 3, 4].includes(flick.kind) || tapKeys.has(key(flick)) || holdStartKeys.has(key(flick)) || holdEndKeys.has(key(flick))) continue;
+    notes.push({ timeSeconds: secondsAt(flick.beat), type: 'regular', isFlick: true });
+  }
+  for (const hold of holds) {
+    const start = hold[0];
+    const end = hold.at(-1);
+    notes.push({ timeSeconds: secondsAt(start.beat), type: 'regular', isFlick: false });
+    // The game creates a continuation judgement every half beat. A visible relay at the same
+    // position remains a separate score judgement.
+    for (let beat = start.beat + 0.5; beat < end.beat - 1e-9; beat += 0.5) {
+      notes.push({ timeSeconds: secondsAt(beat), type: 'mid', isFlick: false });
+    }
+    notes.push({ timeSeconds: secondsAt(end.beat), type: 'regular', isFlick: false });
+  }
+  // Visible relay ticks are score judgements even if the editor stream contains an unmatched visual segment.
+  for (const relay of visibleRelays) notes.push({ timeSeconds: secondsAt(relay.beat), type: 'mid', isFlick: false });
+  notes.sort((left, right) => left.timeSeconds - right.timeSeconds || (left.type === 'mid' ? 1 : -1));
+  const expectedRegular = Number(headers.get('NORMAL_NOTE_COUNT')) + Number(headers.get('FLICK_NOTE_COUNT')) +
+    Number(headers.get('LONG_START_NOTE_COUNT')) + Number(headers.get('LONG_END_NOTE_COUNT')) + Number(headers.get('LONG_FLICK_END_NOTE_COUNT'));
+  // SUS may encode a directional overlay as a second event at the exact same instant. It is one
+  // judgement in Hololive Dreams; retain the header as the authoritative combo count.
+  for (let index = notes.length - 1; notes.filter((note) => note.type === 'regular').length > expectedRegular && index >= 0; index -= 1) {
+    if (notes[index].type === 'regular') notes.splice(index, 1);
+  }
+  const expectedMid = Number(headers.get('LONG_RELAY_NOTE_COUNT')) + Number(headers.get('LONG_CONTINUE_NOTE_COUNT'));
+  // Some editor streams encode a sub-half-beat terminal continuation. Preserve its score category
+  // even when the compact timeline cannot distinguish it from the neighbouring hold endpoint.
+  while (notes.filter((note) => note.type === 'mid').length < expectedMid) {
+    const anchor = notes.at(-1);
+    notes.push({ timeSeconds: anchor?.timeSeconds ?? 0, type: 'mid', isFlick: false });
+  }
+  notes.sort((left, right) => left.timeSeconds - right.timeSeconds || (left.type === 'mid' ? 1 : -1));
+  return { songId: headers.get('MUSIC_ID'), notes, specialStartSeconds: specialStarts.sort((left, right) => left - right).map(secondsAt) };
+}
+
+function readMaster(locale, filename) {
+  const repository = path.resolve(`data/raw/holodoridb/${locale}`);
+  return JSON.parse(execFileSync('git', ['-c', `safe.directory=${repository.replaceAll('\\', '/')}`, '-C', repository, 'show', `origin/main:${filename}`], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }));
+}
+const music = new Map(readMaster('kor', 'Music.json').map((row) => { const data = row.data ?? row; return [data.id, data]; }));
+const translation = new Map(readMaster('eng', 'LangMusic_Eng.json').map((row) => { const data = row.data ?? row; return [data.id, data.text]; }));
+const levels = new Map(readMaster('kor', 'MusicDifficulty.json').map((row) => { const data = row.data ?? row; return [`${data.musicId}.${data.difficultyType.split('_').at(-1).toLowerCase()}`, data.difficultyLevel]; }));
+const songIds = process.argv.slice(2);
+if (songIds.length === 0) throw new Error('곡 ID를 지정하세요.');
+const singerOverrides = { m0527: 'group' };
+const generatedCatalogEntries = [];
+for (const songId of songIds) for (const difficulty of difficulties) {
+  const file = path.resolve(`data/raw/sus/chart_${songId}_${difficulty}.sus`);
+  const bytes = await readFile(file);
+  const timeline = parseSusTimeline(bytes.toString('utf8'));
+  const weight = susScoreWeight(bytes.toString('utf8'));
+  if (timeline.songId !== songId || timeline.notes.length !== weight.fullComboNoteCount) {
+    const mids = timeline.notes.filter((note) => note.type === 'mid').length;
+    throw new Error(`${songId}.${difficulty}: 파서 노트 수 ${timeline.notes.length}/${weight.fullComboNoteCount} 불일치 (중간 ${mids}, 헤더 ${weight.counts.LONG_RELAY + weight.counts.LONG_CONTINUE})`);
+  }
+  const master = music.get(songId);
+  const difficultyLevel = levels.get(`${songId}.${difficulty}`);
+  const susSha256 = createHash('sha256').update(bytes).digest('hex');
+  const formula = formulaScoreMetadata(songId, difficulty, difficultyLevel, timeline.notes.length, master.liveScoreCoefficientPermil, susSha256);
+  const output = { schemaVersion: 1, songId, difficulty, durationSeconds: master.playingSeconds, scoreRatio: formula?.scoreRatio ?? null, scoreCalibrationSource: formula?.scoreCalibrationSource ?? null, ...(formula ?? {}), notes: timeline.notes, specialStartSeconds: timeline.specialStartSeconds, singerType: singerOverrides[songId] ?? master.musicSingerType?.split('_').at(-1).toLowerCase() ?? null, singerCharacterIds: master.characterIds ?? [], source: { page: 'Hololive Dreams CDN via holodori-asset-tools', transcriptionStatus: 'exact_from_decrypted_sus', susSha256 } };
+  await writeFile(path.resolve(`static/data/charts/${songId}.${difficulty}.json`), JSON.stringify(output));
+  generatedCatalogEntries.push({ songId, title: translation.get(master.titleLangId) ?? songId, difficulty, difficultyLevel, durationSeconds: master.playingSeconds, scoreRatio: output.scoreRatio, scoreCalibrationSource: output.scoreCalibrationSource, scoreFormula: output.scoreFormula, totalNotes: output.notes.length, singerType: output.singerType, singerCharacterIds: output.singerCharacterIds, releaseTime: Number(master.startTime), path: `/data/charts/${songId}.${difficulty}.json` });
+  console.log(`${songId}.${difficulty}: ${timeline.notes.length} notes ${timeline.specialStartSeconds.join(', ')}`);
+}
+const catalogPath = path.resolve('src/lib/data/exact-chart-catalog.json');
+const catalog = JSON.parse(await readFile(catalogPath, 'utf8'));
+const generatedKeys = new Set(generatedCatalogEntries.map((entry) => `${entry.songId}.${entry.difficulty}`));
+catalog.charts = [...catalog.charts.filter((entry) => !generatedKeys.has(`${entry.songId}.${entry.difficulty}`)), ...generatedCatalogEntries]
+  .sort((left, right) => left.songId.localeCompare(right.songId) || left.difficulty.localeCompare(right.difficulty));
+await writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`);

scripts/validate-song-score-formula.mjs

@@ -12,7 +12,7 @@ const susReference = await read('docs/research/sus-score-formulas.json');
 const calibrations = await read('data/score-calibrations.json');
 const catalog = await read('src/lib/data/exact-chart-catalog.json');
 const key = (row) => `${row.songId}.${row.difficulty}`;
-const referenceRows = [...reference.charts, ...susReference.charts];
+const referenceRows = [...reference.charts, ...susReference.charts, ...susReference.validationCharts.filter((row) => row.fullComboNoteCount)];
 const byKey = new Map(referenceRows.map((row) => [key(row), row]));
 assert.equal(byKey.size, referenceRows.length, 'Duplicate reference chart');