import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; import { fileURLToPath } from 'node:url'; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const rawRoot = path.join(projectRoot, 'data', 'raw', 'holodoridb'); const outputRoot = path.join(projectRoot, 'data', 'holodoridb'); const appDataRoot = path.join(projectRoot, 'src', 'lib', 'data'); const cardOverridesPath = path.join(projectRoot, 'data', 'card-overrides.csv'); function parseCsvLine(line) { const values = []; let value = ''; let quoted = false; for (let index = 0; index < line.length; index += 1) { const character = line[index]; if (character === '"') { if (quoted && line[index + 1] === '"') { value += '"'; index += 1; } else { quoted = !quoted; } } else if (character === ',' && !quoted) { values.push(value.trim()); value = ''; } else { value += character; } } values.push(value.trim()); return values; } async function loadCardOverrides() { const csv = await readFile(cardOverridesPath, 'utf8'); const lines = csv.replace(/^\uFEFF/, '').split(/\r?\n/).filter((line) => line.trim() && !line.trimStart().startsWith('#')); const headers = parseCsvLine(lines.shift() ?? ''); const cardIdIndex = headers.indexOf('cardId'); const versionNameIndex = headers.indexOf('versionName'); const displayNameIndex = headers.indexOf('displayName'); if (cardIdIndex < 0 || versionNameIndex < 0 || displayNameIndex < 0) { throw new Error('data/card-overrides.csv must contain cardId, versionName, and displayName columns'); } return new Map(lines.map((line) => { const values = parseCsvLine(line); return [values[cardIdIndex], { versionName: values[versionNameIndex] ?? '', displayName: values[displayNameIndex] ?? '' }]; })); } const sources = { eng: { repository: 'https://github.com/HolodoriDB/holodori-db-eng-diff.git', suffix: 'Eng' }, kor: { repository: 'https://github.com/HolodoriDB/holodori-db-kor-diff.git', suffix: 'Kor' }, jpn: { repository: 'https://github.com/HolodoriDB/holodori-db-jpn-diff.git', suffix: 'Jpn' } }; // Tables needed to reconstruct every card and every effect exposed by a card. const masterTableNames = [ 'Card', 'CardLevel', 'CardLevelLimit', 'CardPotential', 'CardPotentialUpgradeItem', 'Character', 'CharacterGrouping', 'CharacterProduction', 'Costume', 'SdCostume', 'LiveLeaderSkill', 'LivePassiveSkill', 'LivePassiveSkillLevel', 'LivePassiveSkillEffect', 'LiveActiveSkill', 'LiveActiveSkillLevel', 'LiveActiveSkillEffect', 'LiveSpecialSkill', 'LiveSpecialSkillLevel', 'LiveSkillTrigger', 'LiveSkillEffectTarget', 'LiveSkillEffectFilterCategory', 'SkillTreeConnectEffect', 'SkillTreeConnectEffectExtent', 'SkillTreeEffect', 'SkillTreeEffectPassiveTrigger', 'SkillTreeEffectTarget', 'SkillTreeEffectValueLimit' ]; const localizationTableNames = [ 'Card', 'Character', 'CharacterGrouping', 'CharacterProduction', 'Costume', 'SdCostume', 'GeneratedLiveActiveSkillEffect', 'GeneratedLiveActiveSkillLevel', 'GeneratedLiveLeaderSkill', 'GeneratedLivePassiveSkillEffect', 'GeneratedLivePassiveSkillLevel', 'GeneratedLiveSkillEffectTarget', 'GeneratedLiveSkillTrigger', 'GeneratedLiveSpecialSkillLevel', 'GeneratedSkillTreeConnectEffect', 'GeneratedSkillTreeEffect', 'GeneratedSkillTreeEffectPassiveTrigger', 'GeneratedSkillTreeEffectTarget', 'LiveSkillEffectFilterCategory', 'SkillTreeEffect', 'SkillTreePoint' ]; function runGit(args, options = {}) { return execFileSync('git', args, { cwd: projectRoot, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'], ...options }).trim(); } function gitAt(repositoryPath, args) { const normalizedPath = repositoryPath.replaceAll('\\', '/'); return runGit(['-c', `safe.directory=${normalizedPath}`, '-C', repositoryPath, ...args]); } async function prepareSource(locale, source) { const repositoryPath = path.join(rawRoot, locale); try { gitAt(repositoryPath, ['rev-parse', '--git-dir']); process.stdout.write(`Updating ${locale} source...\n`); gitAt(repositoryPath, ['fetch', '--depth', '1', 'origin', 'main']); } catch { process.stdout.write(`Cloning ${locale} source...\n`); await mkdir(path.dirname(repositoryPath), { recursive: true }); runGit(['clone', '--depth', '1', source.repository, repositoryPath]); } const commit = gitAt(repositoryPath, ['rev-parse', 'origin/main']); const commitDate = gitAt(repositoryPath, ['show', '-s', '--format=%cI', commit]); const commitMessage = gitAt(repositoryPath, ['show', '-s', '--format=%s', commit]); return { ...source, repositoryPath, commit, commitDate, commitMessage }; } function readSourceFile(source, filename) { return gitAt(source.repositoryPath, ['show', `${source.commit}:${filename}`]); } function parseSourceTable(source, filename) { const text = readSourceFile(source, filename); try { const value = JSON.parse(text); if (!Array.isArray(value)) { throw new Error('top-level value is not an array'); } return { value, text }; } catch (error) { throw new Error(`Invalid JSON in ${filename} from ${source.commit}: ${error.message}`); } } function sha256(value) { return createHash('sha256').update(value).digest('hex'); } function unwrap(row) { return row?.data ?? row; } function indexBy(rows, property = 'id') { return new Map(rows.map((row) => { const value = unwrap(row); return [value[property], value]; })); } function groupBy(rows, property) { const groups = new Map(); for (const row of rows) { const value = unwrap(row); const key = value[property]; const group = groups.get(key) ?? []; group.push(value); groups.set(key, group); } return groups; } function sortByNumber(values, property) { return [...values].sort((left, right) => Number(left[property] ?? 0) - Number(right[property] ?? 0)); } function compactEnum(value) { if (typeof value !== 'string') return value; return value.split('_').at(-1)?.toLowerCase() ?? value; } function buildTextLookups(localizations) { return Object.fromEntries(Object.entries(localizations).map(([locale, tables]) => { const entries = Object.values(tables) .flat() .map(unwrap) .filter((row) => typeof row.id === 'string' && typeof row.text === 'string') .map((row) => [row.id, row.text]); return [locale, new Map(entries)]; })); } function translateId(textLookups, id) { if (!id) return {}; return Object.fromEntries(Object.entries(textLookups).map(([locale, lookup]) => [locale, lookup.get(id) ?? null])); } function withTranslations(value, textLookups) { if (!value) return null; const translations = {}; for (const [key, entry] of Object.entries(value)) { if (key.endsWith('LangId') && typeof entry === 'string') { translations[key.slice(0, -6)] = translateId(textLookups, entry); } if (key.endsWith('LangIds') && Array.isArray(entry)) { translations[key.slice(0, -7)] = entry.map((id) => translateId(textLookups, id)); } } return Object.keys(translations).length > 0 ? { ...value, translations } : { ...value }; } function resolveEffects(rows, targetsById, textLookups) { return rows.map((effect) => ({ ...withTranslations(effect, textLookups), target: effect.liveSkillEffectTargetId ? withTranslations(targetsById.get(effect.liveSkillEffectTargetId), textLookups) : null })); } function assert(condition, message) { if (!condition) throw new Error(`Validation failed: ${message}`); } const preparedSources = {}; for (const [locale, source] of Object.entries(sources)) { preparedSources[locale] = await prepareSource(locale, source); } const masterVersions = new Set( Object.values(preparedSources).map((source) => source.commitMessage.replace(/^Master data version\s+/, '')) ); assert(masterVersions.size === 1, 'locale repositories are not on the same master data version'); const tables = {}; const tableManifest = {}; for (const tableName of masterTableNames) { const filename = `${tableName}.json`; const canonical = parseSourceTable(preparedSources.eng, filename); const hashes = {}; for (const [locale, source] of Object.entries(preparedSources)) { const sourceText = readSourceFile(source, filename); hashes[locale] = sha256(sourceText); } assert(new Set(Object.values(hashes)).size === 1, `${filename} differs between locale repositories`); tables[tableName] = canonical.value; tableManifest[tableName] = { filename, count: canonical.value.length, sha256: hashes.eng }; } const localizations = {}; const localizationManifest = {}; for (const [locale, source] of Object.entries(preparedSources)) { localizations[locale] = {}; localizationManifest[locale] = {}; for (const tableName of localizationTableNames) { const filename = `Lang${tableName}_${source.suffix}.json`; const parsed = parseSourceTable(source, filename); localizations[locale][tableName] = parsed.value; localizationManifest[locale][tableName] = { filename, count: parsed.value.length, sha256: sha256(parsed.text) }; } } const textLookups = buildTextLookups(localizations); const cards = tables.Card.map(unwrap); const charactersById = indexBy(tables.Character); const cardLevelsByGroup = groupBy(tables.CardLevel, 'groupId'); const cardLevelLimitsByGroup = groupBy(tables.CardLevelLimit, 'groupId'); const cardPotentialsByGroup = groupBy(tables.CardPotential, 'groupId'); const passiveSkillsById = indexBy(tables.LivePassiveSkill); const passiveLevelsBySkill = groupBy(tables.LivePassiveSkillLevel, 'livePassiveSkillId'); const passiveEffectsByGroup = groupBy(tables.LivePassiveSkillEffect, 'groupId'); const activeSkillsById = indexBy(tables.LiveActiveSkill); const activeLevelsBySkill = groupBy(tables.LiveActiveSkillLevel, 'liveActiveSkillId'); const activeEffectsByGroup = groupBy(tables.LiveActiveSkillEffect, 'groupId'); const specialSkillsById = indexBy(tables.LiveSpecialSkill); const specialLevelsBySkill = groupBy(tables.LiveSpecialSkillLevel, 'liveSpecialSkillId'); const triggersByGroup = groupBy(tables.LiveSkillTrigger, 'groupId'); const effectTargetsById = indexBy(tables.LiveSkillEffectTarget); const connectEffectsById = groupBy(tables.SkillTreeConnectEffect, 'id'); const connectExtentsByGroup = groupBy(tables.SkillTreeConnectEffectExtent, 'groupId'); const costumesById = indexBy(tables.Costume); const sdCostumesById = indexBy(tables.SdCostume); const leaderSkillsById = indexBy(tables.LiveLeaderSkill); function resolvePassiveSkill(skillId) { const skill = passiveSkillsById.get(skillId); if (!skill) return null; return { ...withTranslations(skill, textLookups), levels: sortByNumber(passiveLevelsBySkill.get(skillId) ?? [], 'level').map((level) => ({ ...withTranslations(level, textLookups), effects: resolveEffects( passiveEffectsByGroup.get(level.livePassiveSkillEffectGroupId) ?? [], effectTargetsById, textLookups ) })) }; } function resolveActiveSkill(skillId) { const skill = activeSkillsById.get(skillId); if (!skill) return null; return { ...withTranslations(skill, textLookups), levels: sortByNumber(activeLevelsBySkill.get(skillId) ?? [], 'level').map((level) => ({ ...withTranslations(level, textLookups), effects: resolveEffects( activeEffectsByGroup.get(level.liveActiveSkillEffectGroupId) ?? [], effectTargetsById, textLookups ), additionalEffects: resolveEffects( activeEffectsByGroup.get(level.additionalLiveActiveSkillEffectGroupId) ?? [], effectTargetsById, textLookups ), triggers: (triggersByGroup.get(level.additionalLiveSkillTriggerGroupId) ?? []) .map((trigger) => withTranslations(trigger, textLookups)) })) }; } function resolveSpecialSkill(skillId) { const skill = specialSkillsById.get(skillId); if (!skill) return null; return { ...withTranslations(skill, textLookups), levels: sortByNumber(specialLevelsBySkill.get(skillId) ?? [], 'level').map((level) => ({ ...withTranslations(level, textLookups), effects: resolveEffects( activeEffectsByGroup.get(level.liveActiveSkillEffectGroupId) ?? [], effectTargetsById, textLookups ), additionalEffects: resolveEffects( activeEffectsByGroup.get(level.additionalLiveActiveSkillEffectGroupId) ?? [], effectTargetsById, textLookups ), triggers: (triggersByGroup.get(level.additionalLiveSkillTriggerGroupId) ?? []) .map((trigger) => withTranslations(trigger, textLookups)) })) }; } function resolveLeaderSkill(skillId) { const skill = leaderSkillsById.get(skillId); if (!skill) return null; return { ...withTranslations(skill, textLookups), effects: resolveEffects( passiveEffectsByGroup.get(skill.livePassiveSkillEffectGroupId) ?? [], effectTargetsById, textLookups ) }; } function resolveCostume(costumeId, costumeIndex) { const costume = costumeIndex.get(costumeId); if (!costume) return null; return { ...withTranslations(costume, textLookups), leaderSkill: costume.liveLeaderSkillId ? resolveLeaderSkill(costume.liveLeaderSkillId) : null }; } const cardRecords = cards .map((card) => { const character = charactersById.get(card.characterId); const connectEffects = sortByNumber(connectEffectsById.get(card.skillTreeConnectEffectId) ?? [], 'level'); return { id: card.id, order: card.order, rarity: compactEnum(card.rarity), attribute: compactEnum(card.attributeType), card: withTranslations(card, textLookups), character: withTranslations(character, textLookups), progression: { levels: sortByNumber(cardLevelsByGroup.get(card.cardLevelGroupId) ?? [], 'level'), levelLimits: sortByNumber(cardLevelLimitsByGroup.get(card.cardLevelLimitGroupId) ?? [], 'levelLimit'), potentials: sortByNumber(cardPotentialsByGroup.get(card.cardPotentialGroupId) ?? [], 'upgradeCount') }, skills: { passive: resolvePassiveSkill(card.livePassiveSkillId), active: resolveActiveSkill(card.liveActiveSkillId), special: resolveSpecialSkill(card.liveSpecialSkillId) }, skillTreeConnectEffects: connectEffects.map((effect) => ({ ...withTranslations(effect, textLookups), extent: sortByNumber( connectExtentsByGroup.get(effect.skillTreeConnectEffectExtentGroupId) ?? [], 'positionY' ) })), rewards: { costume: resolveCostume(card.rewardCostumeId, costumesById), sdCostume: resolveCostume(card.rewardSdCostumeId, sdCostumesById), potentialAlternativeReward: card.potentialAlternativeReward ?? null, acquireReward: card.acquireReward ?? null } }; }) .sort((left, right) => Number(left.order) - Number(right.order)); assert(cards.length > 0, 'no cards were collected'); assert(new Set(cards.map((card) => card.id)).size === cards.length, 'card IDs are not unique'); assert(cards.every((card) => charactersById.has(card.characterId)), 'one or more cards have no character'); assert(cards.every((card) => passiveSkillsById.has(card.livePassiveSkillId)), 'one or more cards have no passive skill'); assert(cards.every((card) => activeSkillsById.has(card.liveActiveSkillId)), 'one or more cards have no active skill'); assert(cards.every((card) => specialSkillsById.has(card.liveSpecialSkillId)), 'one or more cards have no special skill'); const cardNameCoverage = Object.fromEntries(Object.entries(textLookups).map(([locale, lookup]) => [ locale, cards.filter((card) => lookup.has(card.nameLangId)).length ])); const rarityCounts = Object.fromEntries( Object.entries(Object.groupBy(cards, (card) => compactEnum(card.rarity))).map(([key, values]) => [key, values.length]) ); const attributeCounts = Object.fromEntries( Object.entries(Object.groupBy(cards, (card) => compactEnum(card.attributeType))).map(([key, values]) => [key, values.length]) ); const generatedAt = new Date().toISOString(); const sourceMetadata = Object.fromEntries(Object.entries(preparedSources).map(([locale, source]) => [locale, { repository: source.repository.replace(/\.git$/, ''), commit: source.commit, commitDate: source.commitDate, masterDataVersion: source.commitMessage.replace(/^Master data version\s+/, '') }])); const manifest = { schemaVersion: 1, generatedAt, source: 'HolodoriDB community master-data diff repositories', sourceLicense: null, sourceLicenseNote: 'No LICENSE file was present in the three source repositories at collection time.', assetsIncluded: false, sources: sourceMetadata, counts: { cards: cards.length, rarities: rarityCounts, attributes: attributeCounts, localizedCardNames: cardNameCoverage }, tables: tableManifest, localizations: localizationManifest }; const masterBundle = { schemaVersion: 1, generatedAt, sources: sourceMetadata, tables, localizations }; const cardBundle = { schemaVersion: 1, generatedAt, sources: sourceMetadata, cards: cardRecords }; const cardRecordsById = new Map(cardRecords.map((card) => [card.id, card])); const cardOverrides = await loadCardOverrides(); for (const cardId of cardOverrides.keys()) { assert(cardRecordsById.has(cardId), `card override references unknown card: ${cardId}`); } function summarizeSkillLevels(levels) { return levels.map((level) => ({ level: level.level, description: level.translations?.description?.kor ?? '', coolTimeSeconds: level.coolTimeMillisecond == null ? null : level.coolTimeMillisecond / 1000, probabilityPercent: level.activationProbabilityPermilMultiply == null ? null : level.activationProbabilityPermilMultiply / 10, durationSeconds: level.effectDurationMillisecond == null ? null : level.effectDurationMillisecond / 1000 })); } function koreanDescriptions(records) { return (records ?? []) .map((record) => record.translations?.description?.kor ?? '') .filter(Boolean) .join(' '); } function summarizeConditionalSkillLevels(levels) { return summarizeSkillLevels(levels).map((summary, index) => { const condition = koreanDescriptions(levels[index].triggers); const baseEffect = koreanDescriptions(levels[index].effects); const conditionalEffect = koreanDescriptions(levels[index].additionalEffects); return { ...summary, condition, effect: condition ? conditionalEffect : baseEffect }; }); } function summarizeSpecialSkillLevels(levels) { return summarizeSkillLevels(levels).map((summary, index) => { const condition = koreanDescriptions(levels[index].triggers); const baseEffect = koreanDescriptions(levels[index].effects); const conditionalEffect = koreanDescriptions(levels[index].additionalEffects); return { ...summary, condition, effect: condition ? [baseEffect, conditionalEffect].filter(Boolean).join(' ') : baseEffect }; }); } const typeLabels = { '1': '큐트', '2': '퓨어', '3': '해피' }; const statLabels = { performancePermilMultiply: '퍼포먼스', techniquePermilMultiply: '테크닉', sensePermilMultiply: '센스' }; const cardSummary = { schemaVersion: 1, masterDataVersion: [...masterVersions][0], cards: cards .map((card) => { const character = charactersById.get(card.characterId); const cardRecord = cardRecordsById.get(card.id); const cardOverride = cardOverrides.get(card.id); const highestStat = Object.entries(statLabels) .sort(([left], [right]) => Number(card[right]) - Number(card[left]))[0][1]; return { id: card.id, memberOrder: character.order, rarity: Number(compactEnum(card.rarity)), type: typeLabels[compactEnum(card.attributeType)], name: cardOverride?.displayName || textLookups.kor.get(character.nameLangId), versionName: cardOverride?.versionName || '', generation: textLookups.kor.get(character.regularCharacterGroupingNameLangId) || '', highestStat, details: { title: textLookups.kor.get(card.nameLangId), passive: summarizeSkillLevels(cardRecord.skills.passive.levels), active: summarizeConditionalSkillLevels(cardRecord.skills.active.levels), special: summarizeSpecialSkillLevels(cardRecord.skills.special.levels), costume: cardRecord.rewards.costume ? { name: cardRecord.rewards.costume.translations?.name?.kor ?? '', description: cardRecord.rewards.costume.leaderSkill?.translations?.description?.kor ?? cardRecord.rewards.costume.leaderSkill?.effects?.[0]?.translations?.description?.kor ?? '' } : null } }; }) .sort((left, right) => right.rarity - left.rarity || left.memberOrder - right.memberOrder || left.id.localeCompare(right.id) ) .map(({ memberOrder: _memberOrder, ...card }) => card) }; await mkdir(outputRoot, { recursive: true }); await mkdir(appDataRoot, { recursive: true }); await Promise.all([ writeFile(path.join(outputRoot, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`), writeFile(path.join(outputRoot, 'card-master.json'), `${JSON.stringify(masterBundle, null, 2)}\n`), writeFile(path.join(outputRoot, 'cards.json'), `${JSON.stringify(cardBundle, null, 2)}\n`), writeFile(path.join(appDataRoot, 'cards-summary.json'), `${JSON.stringify(cardSummary, null, 2)}\n`) ]); process.stdout.write(`Collected ${cards.length} cards.\n`); process.stdout.write(`Rarities: ${JSON.stringify(rarityCounts)}\n`); process.stdout.write(`Localized names: ${JSON.stringify(cardNameCoverage)}\n`); process.stdout.write(`Output: ${outputRoot}\n`);