commit 207f3f4c96501d8f949f86fb665fee43825ac7dd Author: amizing25 Date: Wed Jun 4 22:52:37 2025 +0700 Initial Commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6cc6a9e --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/target +/scripts +.vscode +out +/testout +/z_scripts diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..d486d28 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2138 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.59.0", +] + +[[package]] +name = "anyhow" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.5.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "common" +version = "0.1.0" +dependencies = [ + "anyhow", + "indicatif", + "reqwest", + "tg-asset-meta", + "tg-bytes-util", + "tracing", + "tracing-subscriber", + "xxhash-rust", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", + "rayon", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "openssl" +version = "0.10.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "program" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "common", + "dashmap", + "rayon", + "serde", + "serde_json", + "tg-bytes-util", + "tg-parser", + "tracing", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "reqwest" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + +[[package]] +name = "rustls-webpki" +version = "0.103.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "socket2" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +dependencies = [ + "fastrand", + "getrandom 0.3.2", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "tg-asset-meta" +version = "0.1.0" +dependencies = [ + "byteorder", + "tg-bytes-util", + "varint-rs", +] + +[[package]] +name = "tg-bytes-util" +version = "0.1.0" +dependencies = [ + "byteorder", + "varint-rs", +] + +[[package]] +name = "tg-parser" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "serde", + "serde_json", + "tg-bytes-util", + "tracing", + "varint-rs", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.44.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.53.0", +] + +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6cf56f6 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,47 @@ +[workspace] +resolver = "3" +members = [ + "crates/bytes-util", + "crates/asset-meta", + "crates/common", + "crates/parser", + "crates/program", +] + +[workspace.package] +version = "0.1.0" + +[workspace.dependencies] +# binary utils +byteorder = "1.5.0" +varint-rs = { version = "2.2.0", features = ["signed"] } +xxhash-rust = { version = "0.8.15", features = ["const_xxh64"] } + +# net utils +reqwest = { version = "0.12.15", features = ["blocking"] } + +# parallelization +dashmap = { version = "6.1.0", features = ["rayon"] } +rayon = "1.10.0" + +# serialization +base64 = "0.22.1" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.140", features = ["preserve_order"] } + +# error handling +anyhow = "1.0.98" + +# loging utils +indicatif = "0.17.11" +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } + +# cli +clap = { version = "4.5.39", features = ["derive"] } + +# local crates +common = { path = "crates/common" } +tg-asset-meta = { path = "crates/asset-meta" } +tg-bytes-util = { path = "crates/bytes-util" } +tg-parser = { path = "crates/parser" } diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d9d27a0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 amizing25 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ff73a8f --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# Traingame parser + +A tool to parse textmap (and maybe other res too) for a certain anime game. + +--- + +## Usages + +### `textmap` + +Processes **only** the textmap files. + +```bash +./program.exe textmap [OPTIONS] +``` + +**Arguments:** + +- `input_url` — URL or path to the persistent data or design bundle +- `output_dir` — Directory where the parsed output will be stored + +**Options:** + +- `--full-textmap` — Parse the entire textmap structure as an array instead of just key-value pairs +- `--save-bytes-file` — Save the `.bytes` files after download + +**Examples:** + +```bash +./program.exe textmap "https://autopatchcn.bhsr.com/design_data/BetaLive/output_10494861_2ed49bac2846_b7f8d02fced269" output/ +``` + +```bash +./program.exe textmap "D:/Star Rail/StarRail_Data/Persistent/DesignData/Windows" output/ +``` + +
+excels + +### `excels` + +Processes the Excel & Textmaps files + +```bash +./program.exe excels [OPTIONS] +``` + +**Arguments:** + +- `data_json` — Path to `data.json` schema +- `excel_path_json` — JSON file that maps Excel types to file paths +- `input_url` — URL or path to the persistent data +- `output_dir` — Output folder for processed files + +**Options:** + +- `--full-textmap` — Enable full textmap parsing if needed for linked data +- `--save-bytes-file` — Save original `.bytes` files +- `--log-error` — Output all encountered errors to the console +- `--config-paths ` — Optional extra config files (in JSON) for parsing additional types + +**Examples:** + +```bash +./program.exe excels data.json excels_path.json https://autopatchcn.bhsr.com/design_data/BetaLive/output_10494861_2ed49bac2846_b7f8d02fced269 output/ --log-error --save-bytes-file +``` + +```bash +./program.exe excels data.json excels_path.json "D:/Star Rail/StarRail_Data/Persistent/DesignData/Windows" output/ --log-error --save-bytes-file +``` + +
+ +
+all + +### `all` + +Processes Textmap, Excels, and Config files + +```bash +./program.exe all [OPTIONS] +``` + +Accepts the **same arguments and options** as the `excels` command. + +**Examples:** + +```bash +./program.exe all data.json excels_path.json "https://autopatchcn.bhsr.com/design_data/BetaLive/output_10494861_2ed49bac2846_b7f8d02fced269" output/ --full-textmap --log-error +``` + +```bash +./program.exe all data.json excels_path.json "D:/Star Rail/StarRail_Data/Persistent/DesignData/Windows" output/ --full-textmap --log-error +``` + +
+ +## Notes + +- For parsing anything other than textmap (i.e., `excels` or `all`), **you must generate `data.json` and `excels_path.json` yourself**. + +--- + +## Credits / References + +- https://arca.live/b/starrailleaks/76183295 +- https://github.com/Hiro420/HSR_Downloader diff --git a/crates/asset-meta/Cargo.toml b/crates/asset-meta/Cargo.toml new file mode 100644 index 0000000..304fcba --- /dev/null +++ b/crates/asset-meta/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "tg-asset-meta" +edition = "2024" +version.workspace = true + +[dependencies] +varint-rs.workspace = true +byteorder.workspace = true + +tg-bytes-util.workspace = true diff --git a/crates/asset-meta/src/design_index.rs b/crates/asset-meta/src/design_index.rs new file mode 100644 index 0000000..674a3f7 --- /dev/null +++ b/crates/asset-meta/src/design_index.rs @@ -0,0 +1,116 @@ +use byteorder::{BE, ReadBytesExt}; +use std::fmt::Write; +use tg_bytes_util::FromBytes; + +#[derive(Debug)] +pub struct DesignIndex { + pub unk_i64: i64, + pub file_count: i32, + pub design_data_count: i32, + pub file_list: Vec, +} + +impl FromBytes for DesignIndex { + fn from_bytes(r: &mut T) -> std::io::Result { + let mut result = DesignIndex { + unk_i64: r.read_i64::()?, + file_count: r.read_i32::()?, + design_data_count: r.read_i32::()?, + file_list: vec![], + }; + + for _ in 0..result.file_count { + result.file_list.push(FileEntry::from_bytes(r)?); + } + + Ok(result) + } +} + +#[derive(Debug)] +pub struct FileEntry { + pub name_hash: i32, + pub file_byte_name: String, + pub size: i64, + pub data_count: i32, + pub data_entries: Vec, + pub unk: u8, +} + +impl FromBytes for FileEntry { + fn from_bytes(r: &mut T) -> std::io::Result { + let mut result = Self { + name_hash: r.read_i32::()?, + file_byte_name: { + let mut buf = vec![0u8; 16]; + r.read_exact(&mut buf)?; + buf.iter().fold(String::with_capacity(16), |mut output, b| { + let _ = output.write_str(&format!("{b:02x}")); + output + }) + }, + size: r.read_i64::()?, + data_count: r.read_i32::()?, + data_entries: vec![], + unk: 0, + }; + + for _ in 0..result.data_count { + result.data_entries.push(DataEntry::from_bytes(r)?); + } + + result.unk = r.read_u8()?; + + Ok(result) + } +} + +#[derive(Debug)] +pub struct DataEntry { + pub name_hash: i32, + pub size: u32, + pub offset: u32, +} + +impl FromBytes for DataEntry { + fn from_bytes(r: &mut T) -> std::io::Result { + Ok(Self { + name_hash: r.read_i32::()?, + size: r.read_u32::()?, + offset: r.read_u32::()?, + }) + } +} + +#[cfg(test)] +mod tests { + use crate::design_index::DesignIndex; + use std::io::Cursor; + use tg_bytes_util::FromBytes; + + #[test] + fn test_parse_design_index() { + const BYTES: &[u8] = include_bytes!("../tests/DesignV.bytes"); + + let mut r = Cursor::new(BYTES); + let parsed = DesignIndex::from_bytes(&mut r).unwrap(); + + assert_eq!(11, parsed.file_count); + assert_eq!(100102, parsed.design_data_count); + assert_eq!(11, parsed.file_list.len()); + + // FileEntry + assert_eq!(-1703948225, parsed.file_list[0].name_hash); + assert_eq!( + "7e3fc08e24890ba15f9c3a8ec1454025", + parsed.file_list[0].file_byte_name.to_string() + ); + assert_eq!(89899, parsed.file_list[0].size); + assert_eq!(1, parsed.file_list[0].data_count); + + // DataEntry + assert_eq!(-1703948225, parsed.file_list[0].data_entries[0].name_hash); + assert_eq!(89899, parsed.file_list[0].data_entries[0].size); + assert_eq!(0, parsed.file_list[0].data_entries[0].offset); + } +} diff --git a/crates/asset-meta/src/lib.rs b/crates/asset-meta/src/lib.rs new file mode 100644 index 0000000..1e2008f --- /dev/null +++ b/crates/asset-meta/src/lib.rs @@ -0,0 +1,2 @@ +pub mod design_index; +pub mod mini_asset; diff --git a/crates/asset-meta/src/mini_asset.rs b/crates/asset-meta/src/mini_asset.rs new file mode 100644 index 0000000..cb92aa9 --- /dev/null +++ b/crates/asset-meta/src/mini_asset.rs @@ -0,0 +1,42 @@ +use std::io::SeekFrom; + +use byteorder::{LE, ReadBytesExt}; +use tg_bytes_util::{ByteHash16, FromBytes}; + +#[derive(Debug)] +pub struct MiniAsset { + pub revision_id: u32, + pub design_index_hash: ByteHash16, +} + +impl FromBytes for MiniAsset { + fn from_bytes(r: &mut T) -> std::io::Result { + r.seek(SeekFrom::Current(6 * 4))?; + Ok(Self { + revision_id: r.read_u32::()?, + design_index_hash: ByteHash16::from_bytes(r)?, + }) + } +} + +#[cfg(test)] +mod tests { + use crate::mini_asset::MiniAsset; + use std::io::Cursor; + use tg_bytes_util::FromBytes; + + #[test] + fn test() { + const BYTES: &[u8] = &[ + 83, 82, 77, 73, 0, 3, 0, 1, 66, 0, 0, 0, 0, 0, 12, 0, 3, 0, 0, 0, 2, 0, 0, 0, 234, 255, + 151, 0, 202, 110, 28, 223, 138, 63, 212, 4, 63, 130, 138, 178, 68, 22, 219, 131, 234, + 55, 0, 0, 0, 0, 0, 0, 210, 249, 237, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let mut r = Cursor::new(BYTES); + let parsed = MiniAsset::from_bytes(&mut r).unwrap(); + assert_eq!( + parsed.design_index_hash.to_string(), + "df1c6eca04d43f8ab28a823f83db1644" + ) + } +} diff --git a/crates/asset-meta/tests/DesignV.bytes b/crates/asset-meta/tests/DesignV.bytes new file mode 100644 index 0000000..4dbd566 Binary files /dev/null and b/crates/asset-meta/tests/DesignV.bytes differ diff --git a/crates/bytes-util/Cargo.toml b/crates/bytes-util/Cargo.toml new file mode 100644 index 0000000..5c23bba --- /dev/null +++ b/crates/bytes-util/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "tg-bytes-util" +edition = "2024" +version.workspace = true + +[dependencies] +varint-rs.workspace = true +byteorder.workspace = true diff --git a/crates/bytes-util/src/from_bytes.rs b/crates/bytes-util/src/from_bytes.rs new file mode 100644 index 0000000..021a104 --- /dev/null +++ b/crates/bytes-util/src/from_bytes.rs @@ -0,0 +1,166 @@ +use std::{ + collections::HashMap, + hash::Hash, + io::{self, Read, Seek}, +}; +use varint_rs::VarintReader; + +pub trait FromBytes: Send + Sync + Sized { + fn from_bytes(r: &mut T) -> io::Result; +} + +impl FromBytes for u8 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_u8_varint() + } +} + +impl FromBytes for u16 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_u16_varint() + } +} + +impl FromBytes for u32 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_u32_varint() + } +} + +impl FromBytes for u64 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_u64_varint() + } +} + +impl FromBytes for usize { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_usize_varint() + } +} + +impl FromBytes for i8 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_i8_varint() + } +} + +impl FromBytes for i16 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_i16_varint() + } +} + +impl FromBytes for i32 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_i32_varint() + } +} + +impl FromBytes for i64 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_i64_varint() + } +} + +impl FromBytes for isize { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + r.read_isize_varint() + } +} + +impl FromBytes for bool { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + Ok(r.read_i8_varint()? != 0) + } +} + +impl FromBytes for f32 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + let mut byte = [0; 4]; + r.read_exact(&mut byte)?; + Ok(f32::from_le_bytes(byte)) + } +} + +impl FromBytes for f64 { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + let mut byte = [0; 8]; + r.read_exact(&mut byte)?; + Ok(f64::from_le_bytes(byte)) + } +} + +impl FromBytes for String { + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + let length = r.read_usize_varint()?; + if length > 1_000_000 { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "attempting to allocate large memory!", + )); + } + let mut buf = vec![0u8; length]; + r.read_exact(&mut buf)?; + Ok(String::from_utf8_lossy(&buf).to_string()) + } +} + +impl FromBytes for Vec +where + T: FromBytes, +{ + #[inline] + fn from_bytes(r: &mut R) -> std::io::Result { + let length = r.read_i32_varint()? as usize; + let mut out = Vec::with_capacity(length); + + for _ in 0..length { + out.push(T::from_bytes(r)?); + } + + Ok(out) + } +} + +impl FromBytes for HashMap +where + K: FromBytes + Eq + Hash, + V: FromBytes, +{ + #[inline] + fn from_bytes(r: &mut T) -> io::Result { + let length = r.read_i32_varint()? as usize; + let mut out = HashMap::with_capacity(length); + + for _ in 0..length { + out.insert(K::from_bytes(r)?, V::from_bytes(r)?); + } + + Ok(out) + } +} + +impl FromBytes for Box +where + T: FromBytes, +{ + #[inline] + fn from_bytes(r: &mut R) -> std::io::Result { + Ok(Box::new(T::from_bytes(r)?)) + } +} diff --git a/crates/bytes-util/src/lib.rs b/crates/bytes-util/src/lib.rs new file mode 100644 index 0000000..1ad8ff9 --- /dev/null +++ b/crates/bytes-util/src/lib.rs @@ -0,0 +1,5 @@ +mod from_bytes; +mod util; + +pub use from_bytes::FromBytes; +pub use util::*; diff --git a/crates/bytes-util/src/util.rs b/crates/bytes-util/src/util.rs new file mode 100644 index 0000000..f89ab34 --- /dev/null +++ b/crates/bytes-util/src/util.rs @@ -0,0 +1,83 @@ +use std::{ + fmt::{Display, Write as _}, + io::Write, +}; +use varint_rs::{VarintReader, VarintWriter}; + +use crate::FromBytes; + +#[derive(Debug)] +pub struct ExistFlag { + data: Vec, + field_length: usize, +} + +impl ExistFlag { + pub fn new( + r: &mut R, + field_length: usize, + ) -> std::io::Result { + let num_varints = field_length.max(1usize).div_ceil(64); + let mut data = Vec::with_capacity(num_varints); + for _ in 0..num_varints { + data.push(r.read_u64_varint()?); + } + Ok(Self { data, field_length }) + } + + pub fn exists(&self, index: usize) -> bool { + if index >= self.field_length { + panic!("out of bound field index: {index}") + } else { + let segment_idx = index / 64; + let bit_idx = index % 64; + ((self.data[segment_idx] >> bit_idx) & 1) != 0 + } + } + + pub fn write(writer: &mut W, exist_flags: &[bool]) -> std::io::Result<()> { + let field_length = exist_flags.len(); + let num_varints = field_length.max(1).div_ceil(64); + let mut data = vec![0u64; num_varints]; + + for (i, &exists) in exist_flags.iter().enumerate() { + let segment_idx = i / 64; + let bit_idx = i % 64; + if exists { + data[segment_idx] |= 1u64 << bit_idx; + } + } + + for val in data { + writer.write_u64_varint(val)?; + } + Ok(()) + } +} + +#[derive(Debug)] +pub struct ByteHash16(Vec); + +impl FromBytes for ByteHash16 { + fn from_bytes(r: &mut T) -> std::io::Result { + let mut full_hash = [0u8; 16]; + for i in 0..4 { + let mut chunk = vec![0u8; 4]; + r.read_exact(&mut chunk)?; + for j in 0..4 { + full_hash[i * 4 + j] = chunk[3 - j]; + } + } + Ok(Self(full_hash.to_vec())) + } +} + +impl Display for ByteHash16 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0.iter().fold(String::new(), |mut output, b| { + let _ = output.write_str(&format!("{b:02x}")); + output + }))?; + Ok(()) + } +} diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml new file mode 100644 index 0000000..416366a --- /dev/null +++ b/crates/common/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "common" +edition = "2024" +version.workspace = true + +[dependencies] +xxhash-rust.workspace = true +reqwest.workspace = true +anyhow.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +indicatif.workspace = true + +tg-asset-meta.workspace = true +tg-bytes-util.workspace = true diff --git a/crates/common/src/downloader.rs b/crates/common/src/downloader.rs new file mode 100644 index 0000000..de94d72 --- /dev/null +++ b/crates/common/src/downloader.rs @@ -0,0 +1,230 @@ +use super::hash; +use anyhow::{Context, Result}; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use reqwest::{StatusCode, blocking::Client}; +use std::{ + collections::HashMap, + io::{Cursor, Read as _}, + path::PathBuf, + thread, + time::Duration, +}; +use tg_asset_meta::{ + design_index::{DesignIndex, FileEntry}, + mini_asset::MiniAsset, +}; +use tg_bytes_util::FromBytes; + +const MAX_RETRIES: usize = 3; +const RETRY_DELAY: Duration = Duration::from_millis(100); + +pub fn download_all_design_data( + design_data_url: String, + output_folder: Option, + filter_hashes: Vec, +) -> Result>> { + let client = Client::new(); + let mp = MultiProgress::new(); + + let mini_asset = download_mini_asset(&client, &design_data_url, &mp, &output_folder) + .context("Failed to download mini asset")?; + + let design_index = + download_design_index(&client, &design_data_url, &mp, &mini_asset, &output_folder) + .context("Failed to download design index")?; + + let mut handles = HashMap::with_capacity(design_index.file_list.len()); + + for file_entry in design_index.file_list { + let client = client.clone(); + let design_data_url = design_data_url.clone(); + let mp = mp.clone(); + let output_folder = output_folder.clone(); + let byte_name = file_entry.file_byte_name.clone(); + + if !filter_hashes.is_empty() + && !file_entry + .data_entries + .iter() + .any(|e| filter_hashes.contains(&e.name_hash)) + { + continue; + } + + handles.insert( + byte_name, + thread::spawn(move || { + let data = download_design_bytes( + &client, + &design_data_url, + &mp, + &file_entry, + &output_folder, + )?; + + // Special handling for ConfigManifest, since they are in JSON format. + if file_entry.name_hash + == hash::get_32bit_hash_const("BakedConfig/ConfigManifest.json") + { + return Result::>>::Ok(HashMap::from([( + file_entry.name_hash, + data, + )])); + }; + + Ok(file_entry + .data_entries + .iter() + .map(|data_entry| { + let slice = &data[data_entry.offset as usize + ..(data_entry.offset + data_entry.size) as usize]; + + (data_entry.name_hash, slice.to_vec()) + }) + .collect::>>()) + }), + ); + } + + let results: HashMap>> = handles + .into_iter() + .filter_map(|(byte_name, handle)| match handle.join() { + Ok(Ok(data)) => Some((byte_name, data)), + Ok(Err(e)) => { + tracing::error!("Download error: {:?}", e); + None + } + Err(e) => { + tracing::error!("Thread panicked: {:?}", e); + None + } + }) + .collect(); + + Ok(results + .into_iter() + .flat_map(|(_, inner)| inner.into_iter()) + .collect()) +} + +#[inline] +fn download_mini_asset( + client: &Client, + design_data_url: &str, + mp: &MultiProgress, + output_folder: &Option, +) -> anyhow::Result { + let res = download_bytes( + client, + &format!("{design_data_url}/client/Windows/M_DesignV.bytes"), + mp, + )?; + let mini_asset = MiniAsset::from_bytes(&mut Cursor::new(&res))?; + + save_file(output_folder, &res, "M_DesignV.bytes"); + + Ok(mini_asset) +} + +#[inline] +fn download_design_index( + client: &Client, + design_data_url: &str, + mp: &MultiProgress, + mini_asset: &MiniAsset, + output_folder: &Option, +) -> Result { + let name = format!("DesignV_{}.bytes", mini_asset.design_index_hash); + let res = download_bytes( + client, + &format!("{design_data_url}/client/Windows/{name}",), + mp, + )?; + + let design_index = DesignIndex::from_bytes(&mut Cursor::new(&res))?; + + save_file(output_folder, &res, &name); + + Ok(design_index) +} + +#[inline] +fn download_design_bytes( + client: &Client, + design_data_url: &str, + mp: &MultiProgress, + file_entry: &FileEntry, + output_folder: &Option, +) -> Result> { + let name = format!("{}.bytes", file_entry.file_byte_name); + let bytes = download_bytes( + client, + &format!("{design_data_url}/client/Windows/{name}",), + mp, + )?; + save_file(output_folder, &bytes, &name); + + Ok(bytes) +} + +#[inline] +fn save_file(output_folder: &Option, bytes: &Vec, file_name: &str) { + if let Some(output_folder) = output_folder { + let output_folder = output_folder.join("DesignData"); + if !output_folder.is_dir() { + let _ = std::fs::create_dir_all(&output_folder); + } + let _ = std::fs::write(output_folder.join(file_name), bytes); + } +} + +fn download_bytes(client: &Client, design_data_url: &str, mp: &MultiProgress) -> Result> { + if !design_data_url.starts_with("http") { + return Ok(std::fs::read( + design_data_url.replace("client/Windows", ""), + )?); + } + + for attempt in 1..=MAX_RETRIES { + let result = (|| -> Result> { + let resp = client.get(design_data_url).send()?; + let status = resp.status(); + if status != StatusCode::OK { + return Err(anyhow::format_err!( + "Server returned non OK code for {design_data_url} {:?}", + status + )); + } + + let total = resp.content_length().unwrap_or(0); + let pb = mp.add(ProgressBar::new(total)); + pb.set_style( + ProgressStyle::with_template("{msg} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, ETA: {eta})")? + .progress_chars("##-"), + ); + + let file_name = design_data_url.split('/').next_back().unwrap_or_default(); + pb.set_message(format!("Downloading {file_name}")); + + let mut reader = pb.wrap_read(resp); + let mut buffer = Vec::with_capacity(total as usize); + reader.read_to_end(&mut buffer)?; + + pb.finish_with_message(format!("Downloaded {file_name}")); + Ok(buffer) + })(); + + match result { + Ok(data) => return Ok(data), + Err(e) if attempt < MAX_RETRIES => { + mp.println(format!( + "Retry {attempt}/{MAX_RETRIES} for {design_data_url} due to error: {e}" + ))?; + std::thread::sleep(RETRY_DELAY); + } + Err(e) => return Err(e), + } + } + + unreachable!() +} diff --git a/crates/common/src/hash.rs b/crates/common/src/hash.rs new file mode 100644 index 0000000..0f4ed0f --- /dev/null +++ b/crates/common/src/hash.rs @@ -0,0 +1,25 @@ +pub const fn get_32bit_hash_const(s: &str) -> i32 { + let mut hash1: i32 = 5381; + let mut hash2: i32 = hash1; + + let bytes = s.as_bytes(); + let length = bytes.len(); + + let mut i = 0; + while i < length { + hash1 = ((hash1 << 5).wrapping_add(hash1)) ^ (bytes[i] as i32); + + if i + 1 < length { + hash2 = ((hash2 << 5).wrapping_add(hash2)) ^ (bytes[i + 1] as i32); + } + + i += 2; + } + + hash1.wrapping_add(hash2.wrapping_mul(1566083941)) +} + +#[inline] +pub fn get_64bit_hash_const(s: &str) -> u64 { + xxhash_rust::const_xxh64::xxh64(s.as_bytes(), 0) +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs new file mode 100644 index 0000000..d2288b5 --- /dev/null +++ b/crates/common/src/lib.rs @@ -0,0 +1,3 @@ +pub mod downloader; +pub mod hash; +pub mod logging; diff --git a/crates/common/src/logging.rs b/crates/common/src/logging.rs new file mode 100644 index 0000000..d9120e6 --- /dev/null +++ b/crates/common/src/logging.rs @@ -0,0 +1,30 @@ +use tracing::Level; +use tracing_subscriber::{ + EnvFilter, Layer, filter, fmt, layer::SubscriberExt as _, util::SubscriberInitExt as _, +}; + +pub fn init(level: tracing::Level) { + tracing_subscriber::fmt() + .with_max_level(level) + .with_env_filter( + EnvFilter::builder() + .with_default_directive(level.into()) + .from_env() + .unwrap() + .add_directive("ureq=error".parse().unwrap()), + ) + .without_time() + .with_target(false) + .init(); +} + +pub fn init_info_only() { + tracing_subscriber::registry() + .with( + fmt::layer() + .without_time() + .with_target(false) + .with_filter(filter::filter_fn(|m| m.level() == &Level::INFO)), + ) + .init(); +} diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml new file mode 100644 index 0000000..617888f --- /dev/null +++ b/crates/parser/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "tg-parser" +edition = "2024" +version.workspace = true + +[dependencies] +varint-rs.workspace = true +anyhow.workspace = true +tracing.workspace = true +serde.workspace = true +serde_json.workspace = true +base64.workspace = true + +tg-bytes-util.workspace = true diff --git a/crates/parser/src/custom_parser.rs b/crates/parser/src/custom_parser.rs new file mode 100644 index 0000000..d5e9954 --- /dev/null +++ b/crates/parser/src/custom_parser.rs @@ -0,0 +1,232 @@ +use std::{ + collections::HashMap, + io::{Read, Seek}, + sync::LazyLock, +}; + +use crate::{DynamicParser, ValueKind}; +use base64::Engine; +use serde_json::{Map, Number, Value, json}; +use varint_rs::VarintReader; + +use tg_bytes_util::FromBytes; + +type CustomParser = + HashMap<&'static str, for<'a> fn(&mut DynamicParser<'a>) -> anyhow::Result>; + +pub static CUSTOM_PARSER: LazyLock = LazyLock::new(|| { + let mut m: CustomParser = HashMap::with_capacity(7); + m.insert("RPG.GameCore.FixPoint", fix_point_parser); + m.insert("RPG.GameCore.DynamicValue", dynamic_value_parser); + m.insert("LAHCFFKCOBC", dynamic_values_parser); + m.insert("RPG.GameCore.DynamicFloat", dynamic_float_parser); + m.insert("RPG.GameCore.ReadInfo", read_info_parser); + m.insert("RPG.GameCore.JsonEnum", json_enum_parser); + m.insert("RPG.Client.TextID", textid_parser); + m +}); + +fn read_bytes(cursor: &mut R, len: usize) -> anyhow::Result> { + let mut buffer = vec![0u8; len]; + cursor.read_exact(&mut buffer)?; + Ok(buffer) +} + +fn read_byte(cursor: &mut R) -> anyhow::Result { + Ok(read_bytes(cursor, 1)?[0]) +} + +fn read_bool(cursor: &mut R) -> anyhow::Result { + Ok(read_byte(cursor)? != 0) +} + +fn fix_point_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + let value = parser.cursor.read_i64_varint()? as f32; + Ok(json!({ + "Value": (value / (2f32).powf(32f32)) as f64 + })) +} + +fn dynamic_value_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + let value_type = parser.cursor.read_i8_varint()?; + + let (r#type, value) = match value_type { + 0 => ( + String::from("Int32"), + Value::Number(i32::from_bytes(&mut parser.cursor)?.into()), + ), + 1 => ( + String::from("Float"), + Value::Number(Number::from_f64(f32::from_bytes(&mut parser.cursor)? as f64).unwrap()), + ), + 2 => ( + String::from("Boolean"), + Value::Bool(bool::from_bytes(&mut parser.cursor)?), + ), + 3 => { + let length = parser.cursor.read_i64_varint()? as usize; + if length > 1_000_000 { + return Err(anyhow::format_err!("attempting to allocate large memory!")); + } + let mut result = Vec::with_capacity(length); + for _ in 0..length { + result.push(parser.parse( + &ValueKind::Class(String::from("RPG.GameCore.DynamicValue")), + false, + )?); + } + (String::from("Array"), serde_json::to_value(result)?) + } + 4 => { + let length = parser.cursor.read_i64_varint()? as usize; + if length > 1_000_000 { + return Err(anyhow::format_err!("attempting to allocate large memory!")); + } + let mut result = Vec::with_capacity(length); + for _ in 0..length { + let _ = parser.cursor.read_i64_varint()?; + let _ = parser.cursor.read_i64_varint()?; + + result.push(parser.parse( + &ValueKind::Class(String::from("RPG.GameCore.DynamicValue")), + false, + )?); + } + (String::from("Map"), serde_json::to_value(result)?) + } + 5 => ( + String::from("String"), + Value::String(String::from_bytes(&mut parser.cursor)?), + ), + _ => (String::from("Null"), Value::Null), + }; + + Ok(json!({ + "Type": r#type, + "Value": value + })) +} + +fn dynamic_values_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + let length = parser.cursor.read_u64_varint()? as usize; + + if length > 1_000_000 { + return Err(anyhow::format_err!("attempting to allocate large memory!")); + } + + let mut floats = Map::with_capacity(length); + + for _ in 0..length { + let key = parser.parse( + &ValueKind::Class(String::from("RPG.GameCore.StringHash")), + false, + )?; + + let v12 = bool::from_bytes(&mut parser.cursor)?; + let value = if v12 { + let v7 = dynamic_float_parser(parser)?; + let v8 = dynamic_float_parser(parser)?; + let v9 = dynamic_float_parser(parser)?; + + let read_info = read_info_parser(parser)?; + json!({ + "v7": v7, + "v8": v8, + "v9": v9, + "ReadInfo": read_info, + }) + } else { + let v24 = fix_point_parser(parser)?; + + let v17 = bool::from_bytes(&mut parser.cursor)?; + let unk = if v17 { + let v15 = fix_point_parser(parser)?; + let v16 = fix_point_parser(parser)?; + json!({ + "v15": v15, + "v16": v16 + }) + } else { + json!({}) + }; + + let read_info = read_info_parser(parser)?; + + json!({ + "ReadInfo": read_info, + "unk": unk, + "v24": v24 + }) + }; + + floats.insert(key.to_string(), value); + } + + Ok(json!({ + "Floats": floats + })) +} + +fn dynamic_float_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + let is_dynamic = read_bool(&mut parser.cursor)?; + + Ok(if is_dynamic { + let opcode_len = read_byte(&mut parser.cursor)? as usize; + let opcodes = base64::engine::general_purpose::STANDARD + .encode(read_bytes(&mut parser.cursor, opcode_len)?); + + let fixed_values = (0..read_byte(&mut parser.cursor)?) + .map(|_| fix_point_parser(parser)) + .collect::, _>>()?; + + let dynamic_hashes = (0..read_byte(&mut parser.cursor)?) + .map(|_| parser.cursor.read_i32_varint()) + .collect::, _>>()?; + + json!({ + "IsDynamic": true, + "PostfixExpr": { + "OpCodes": opcodes, + "FixedValues": fixed_values, + "DynamicHashes": dynamic_hashes + } + }) + } else { + let fixed_value = fix_point_parser(parser)?; + + json!({ + "IsDynamic": false, + "FixedValue": fixed_value + }) + }) +} + +fn read_info_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + let has_read_info = read_bool(&mut parser.cursor)?; + + if has_read_info { + let string = String::from_bytes(&mut parser.cursor)?; + let v17 = parser.cursor.read_i64_varint()?; + + Ok(json!({ + "AKFKONMJCEC": string, + "EGMAFIOOKJJ": v17 + })) + } else { + Ok(Value::Null) + } +} + +fn json_enum_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + Ok(json!({ + "EnumIndex": parser.cursor.read_i32_varint()?, + "Value": parser.cursor.read_i32_varint()? + })) +} + +fn textid_parser<'a>(parser: &mut DynamicParser<'a>) -> anyhow::Result { + Ok(json!({ + "Hash": parser.cursor.read_i32_varint()?, + "Hash64": parser.cursor.read_u64_varint()? + })) +} diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs new file mode 100644 index 0000000..f15e37a --- /dev/null +++ b/crates/parser/src/lib.rs @@ -0,0 +1,365 @@ +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::io::Cursor; + +use anyhow::Context; +use serde::Deserialize; +use serde_json::json; +use serde_json::{Map, Value}; +use tg_bytes_util::{ExistFlag, FromBytes}; +use varint_rs::VarintReader; +mod custom_parser; + +#[derive(Debug, Deserialize)] +pub enum DataDefine { + Class { + skip_existflag_check: Option, + fields: Vec, + interfaces: Vec, + }, + Struct { + fields: Vec, + interfaces: Vec, + }, + Typeindex { + base: String, + descendants: BTreeMap, + }, + Enum(String, BTreeMap), +} + +#[derive(Debug, Deserialize)] +pub struct DataField { + pub field_name: String, + pub data_type: ValueKind, +} + +#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +pub enum ValueKind { + Primitive(String), + Array(Box), + Dictionary(Box, Box), + Class(String), + Other(), +} + +pub struct DynamicParser<'a> { + pub types: &'a HashMap, + pub cursor: Cursor<&'a Vec>, +} + +impl<'a> DynamicParser<'a> { + pub fn new(types: &'a HashMap, data: &'a Vec) -> Self { + Self { + types, + cursor: Cursor::new(data), + } + } + + pub fn parse(&mut self, kind: &ValueKind, include_type: bool) -> anyhow::Result { + if self.remaining() < 1 { + tracing::debug!("{:?} buffer is empty", kind); + + return Ok(match kind { + ValueKind::Primitive(_) => Value::Number(0.into()), + ValueKind::Array(_) => Value::Array(Vec::with_capacity(0)), + ValueKind::Dictionary(_, _) | ValueKind::Class(_) => { + Value::Object(Map::with_capacity(0)) + } + ValueKind::Other() => Value::Null, + }); + } + + Ok(match kind { + ValueKind::Primitive(cs_type) => match cs_type.as_str() { + "byte" => Value::Number(self.cursor.read_u8_varint()?.into()), + "sbyte" => Value::Number(self.cursor.read_i8_varint()?.into()), + "short" => Value::Number(self.cursor.read_i16_varint()?.into()), + "ushort" => Value::Number(self.cursor.read_u16_varint()?.into()), + "int" => Value::Number(self.cursor.read_i32_varint()?.into()), + "uint" => Value::Number(self.cursor.read_u32_varint()?.into()), + "long" => Value::Number(self.cursor.read_i64_varint()?.into()), + "ulong" => Value::Number(serde_json::Number::from(self.cursor.read_u64_varint()?)), + "float" => { + let raw = f32::from_bytes(&mut self.cursor)? as f64; + let sanitized = if raw.is_finite() { raw } else { 0.0 }; + let number = serde_json::Number::from_f64(sanitized) + .ok_or_else(|| anyhow::anyhow!("float should always be finite"))?; + Value::Number(number) + } + "double" => Value::Number( + serde_json::Number::from_f64(f64::from_bytes(&mut self.cursor)?) + .ok_or_else(|| anyhow::anyhow!("invalid double"))?, + ), + "bool" => Value::Bool(bool::from_bytes(&mut self.cursor)?), + "string" => Value::String(String::from_bytes(&mut self.cursor)?), + other => return Err(anyhow::format_err!("unhandled primitive: {other}")), + }, + ValueKind::Dictionary(key, value) => { + tracing::debug!( + "ValueKind::Dictionary(cursor_pos: {}) -> Dictionary<{:?}, {:?}>", + self.cursor.position(), + key, + value + ); + + let length = self.cursor.read_i64_varint()? as usize; + + tracing::debug!( + "ValueKind::Dictionary(cursor_pos: {}) -> Dictionary length: {}", + self.cursor.position(), + length + ); + + if length > 1_000_000 { + return Err(anyhow::format_err!("attempting to allocate large memory!")); + } + + let mut output = Map::with_capacity(length); + + for _ in 0..length { + let key = self.parse(key, false)?; + output.insert( + if let Value::String(s) = key { + s + } else { + key.to_string() + }, + self.parse(value, false)?, + ); + } + + Value::Object(output) + } + ValueKind::Array(value) => { + tracing::debug!( + "ValueKind::Array(cursor_pos: {}) -> {:?}[]", + self.cursor.position(), + value + ); + + let length = self.cursor.read_i64_varint()? as usize; + + tracing::debug!( + "ValueKind::Array(cursor_pos: {}) -> Array length: {}", + self.cursor.position(), + length + ); + + if length > 1_000_000 { + return Err(anyhow::format_err!("attempting to allocate large memory!")); + } + + let mut output = Vec::with_capacity(length); + + for _ in 0..length { + output.push(self.parse(value, false)?); + } + + Value::Array(output) + } + ValueKind::Class(class_name) => { + tracing::debug!( + "ValueKind::Class(cursor_pos: {}) -> {}", + self.cursor.position(), + class_name + ); + + if let Some(custom) = custom_parser::CUSTOM_PARSER.get(class_name.as_str()) { + return custom(self); + } + + let Some(define) = self.types.get(class_name) else { + return Err(anyhow::format_err!("unhandled type: {}", &class_name)); + }; + + let mut result = self.parse_class_kind(define)?; + + if include_type { + result.as_object_mut().and_then(|f| { + f.shift_insert( + 0, + "$type".into(), + Value::String( + class_name + .strip_suffix("Inner") + .unwrap_or(class_name) + .to_string(), + ), + ) + }); + } + + result + } + _ => return Err(anyhow::format_err!("unknown data kind!")), + }) + } + + fn parse_class_kind(&mut self, data_type: &DataDefine) -> anyhow::Result { + Ok(match data_type { + DataDefine::Class { + skip_existflag_check, + fields, + interfaces: _, + } => { + if skip_existflag_check.is_some() { + return Ok(json!({})); + } + + let exist_flag = ExistFlag::new(&mut self.cursor, fields.len())?; + let mut output = Map::with_capacity(fields.len()); + for (i, field) in fields.iter().enumerate() { + if exist_flag.exists(i) { + tracing::debug!( + "DataDefine::Class(cursor_pos: {}) -> Key: {}", + self.cursor.position(), + field.field_name + ); + + let value = self.parse(&field.data_type, false)?; + + tracing::debug!( + "DataDefine::Class(cursor_pos: {}) -> Value: {:?}", + self.cursor.position(), + value + ); + + output.insert(field.field_name.to_string(), value); + } else { + tracing::debug!( + "DataDefine::Class(cursor_pos: {}) -> Field not exist! key: {}", + self.cursor.position(), + field.field_name + ); + } + } + Value::Object(output) + } + DataDefine::Struct { + fields, + interfaces: _, + } => { + let mut output = Map::with_capacity(fields.len()); + for field in fields { + tracing::debug!( + "DataDefine::Struct(cursor_pos: {}) -> Key: {}", + self.cursor.position(), + field.field_name + ); + + let value = self.parse(&field.data_type, false)?; + + tracing::debug!( + "DataDefine::Struct(cursor_pos: {}) -> Value: {:?}", + self.cursor.position(), + value + ); + + output.insert(field.field_name.to_string(), value); + } + Value::Object(output) + } + DataDefine::Typeindex { base, descendants } => { + tracing::debug!( + "DataDefine::Typeindex(cursor_pos: {})", + self.cursor.position() + ); + + let typeindex = self + .cursor + .read_u64_varint() + .context("typeindex reading failed")?; + + let Some(descendant) = descendants.get(&typeindex) else { + return Err(anyhow::format_err!( + "typeindex not exist! dict: {:?} type index: {}", + descendants, + typeindex + )); + }; + + if let ValueKind::Class(descendant) = descendant + && let Some(DataDefine::Typeindex { + base: _, + descendants, + }) = self.types.get(descendant) + && let Some(descendant) = descendants.get(&0) + { + return self.parse(descendant, true); + } + + tracing::debug!( + "DataDefine::Typeindex(cursor_pos: {}) -> {} typeindex: {typeindex}", + self.cursor.position(), + base + ); + + return self.parse(descendant, true); + } + DataDefine::Enum(enum_type, enums) => { + let enum_value = match enum_type.as_str() { + "ulong" => { + let discriminant = self.cursor.read_u64_varint()?; + if let Some(enum_value) = enums.get(&discriminant.to_string()) { + enum_value + } else { + tracing::debug!( + "enum_value not exist! enums: {:?} discriminant: {}", + enums, + discriminant + ); + &discriminant.to_string() + } + } + "int" => { + let discriminant = self.cursor.read_i32_varint()?; + if let Some(discriminant) = enums.get(&discriminant.to_string()) { + discriminant + } else { + tracing::warn!( + "enum_value not exist! enums: {:?} discriminant: {}", + enums, + discriminant + ); + &format!("{discriminant}") + } + } + "uint" => { + let discriminant = self.cursor.read_u32_varint()?; + if let Some(discriminant) = enums.get(&discriminant.to_string()) { + discriminant + } else { + tracing::warn!( + "enum_value not exist! enums: {:?} discriminant: {}", + enums, + discriminant + ); + &format!("{discriminant}") + } + } + "ushort" => { + let discriminant = self.cursor.read_u16_varint()?; + if let Some(discriminant) = enums.get(&discriminant.to_string()) { + discriminant + } else { + tracing::debug!( + "enum_value not exist! enums: {:?} discriminant: {}", + enums, + discriminant + ); + &format!("{discriminant}") + } + } + _ => return Err(anyhow::format_err!("unsupported enum type: {}", enum_type)), + }; + Value::String(enum_value.into()) + } + }) + } + + #[inline] + fn remaining(&self) -> usize { + self.cursor.get_ref().len() - self.cursor.position() as usize + } +} diff --git a/crates/program/Cargo.toml b/crates/program/Cargo.toml new file mode 100644 index 0000000..439ca1a --- /dev/null +++ b/crates/program/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "program" +edition = "2024" +version.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true +anyhow.workspace = true +rayon.workspace = true +dashmap.workspace = true +common.workspace = true +clap.workspace = true + +tg-parser.workspace = true +tg-bytes-util.workspace = true diff --git a/crates/program/src/actions/config/adventure_ability.rs b/crates/program/src/actions/config/adventure_ability.rs new file mode 100644 index 0000000..7c6a648 --- /dev/null +++ b/crates/program/src/actions/config/adventure_ability.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.AdventureAbilityConfigList +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .adventure_ability_config + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.AdventureAbilityConfigList", + assets, + types, + out_folder + ); + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/adventure_modifier.rs b/crates/program/src/actions/config/adventure_modifier.rs new file mode 100644 index 0000000..bcfc9ba --- /dev/null +++ b/crates/program/src/actions/config/adventure_modifier.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.AdventureModifierLookupTable +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .adventure_modifier_config + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.AdventureModifierLookupTable", + assets, + types, + out_folder + ); + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/complex_skill_ai_global.rs b/crates/program/src/actions/config/complex_skill_ai_global.rs new file mode 100644 index 0000000..ba968b4 --- /dev/null +++ b/crates/program/src/actions/config/complex_skill_ai_global.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.ComplexSkillAIGlobalGroupLookup +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .complex_skill_aiglobal_group_config + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.ComplexSkillAIGlobalGroupLookup", + assets, + types, + out_folder + ); + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/config_ability.rs b/crates/program/src/actions/config/config_ability.rs new file mode 100644 index 0000000..5efd90a --- /dev/null +++ b/crates/program/src/actions/config/config_ability.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.TurnBasedAbilityConfigList +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .turn_based_ability_config + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.TurnBasedAbilityConfigList", + assets, + types, + out_folder + ); + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/global_modifier.rs b/crates/program/src/actions/config/global_modifier.rs new file mode 100644 index 0000000..4106886 --- /dev/null +++ b/crates/program/src/actions/config/global_modifier.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.GlobalModifierConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .global_modifier_config + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.GlobalModifierConfig", + assets, + types, + out_folder + ); + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/global_task_template.rs b/crates/program/src/actions/config/global_task_template.rs new file mode 100644 index 0000000..3d87851 --- /dev/null +++ b/crates/program/src/actions/config/global_task_template.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.GlobalTaskListTemplateConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .global_task_template + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.GlobalTaskListTemplateConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/level_output.rs b/crates/program/src/actions/config/level_output.rs new file mode 100644 index 0000000..7bb5825 --- /dev/null +++ b/crates/program/src/actions/config/level_output.rs @@ -0,0 +1,133 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde_json::{Map, Value}; +use std::{collections::HashMap, fs, path::Path}; +use tg_parser::DataDefine; + +fn parse_floor( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, +) -> Result<()> { + let maze_plane: Vec> = + serde_json::from_slice(&fs::read(out_folder.join("ExcelOutput/MazePlane.json"))?)?; + + // Flatten plane and floor ID tuples + let paths: Vec<_> = maze_plane + .iter() + .flat_map(|p| { + let list = p + .get("FloorIDList") + .unwrap() + .as_array() + .unwrap() + .iter() + .map(|v| v.as_u64().unwrap() as u32) + .collect::>(); + let plane_id = p.get("PlaneID").unwrap().as_u64().unwrap() as u32; + list.into_iter().map(move |f| (plane_id, f)) + }) + .collect(); + + paths.par_iter().for_each(|(plane_id, floor_id)| { + let name = format!("P{plane_id}_F{floor_id}"); + + let configs = [ + ( + format!("Config/LevelOutput/RuntimeFloor/{name}.json"), + "RPG.GameCore.RtLevelFloorInfo", + ), + ( + format!("Config/LevelOutput_Baked/Floor/{name}_Baked.json"), + "RPG.GameCore.LevelFloorBakedInfo", + ), + ( + format!( + "Config/LevelOutput_Baked/FloorCrossMapBriefInfo/CrossMapBriefInfo_{name}.json" + ), + "RPG.GameCore.LevelFloorCrossMapBriefInfo", + ), + ( + format!("Config/LevelOutput/Region/FloorRegion_{name}.json"), + "RPG.GameCore.LevelRegionInfos", + ), + ( + format!("Config/LevelOutput/RotatableRegion/RotatableRegion_Floor_{floor_id}.json"), + "RPG.GameCore.MapRotationConfig", + ), + ( + format!("Config/LevelOutput/EraFlipper/EraFlipper_Floor_{floor_id}.json"), + "RPG.GameCore.EraFlipperConfig", + ), + ( + format!("Config/LevelOutput/Map/MapInfo_{name}.json"), + "RPG.GameCore.LevelNavmapConfig", + ), + ]; + + for (path, type_name) in configs { + parse_and_count!(&path, type_name, assets, types, out_folder) + } + }); + + Ok(()) +} + +fn parse_group( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, +) -> Result<()> { + let runtime_floor = fs::read_dir(out_folder.join("Config/LevelOutput/RuntimeFloor"))?; + + let mut group_paths = Vec::new(); + + for floor in runtime_floor { + let Ok(entry) = floor else { + continue; + }; + + let Ok(slice) = fs::read(entry.path()) else { + continue; + }; + + let json: Value = serde_json::from_slice(&slice)?; + json.as_object() + .and_then(|v| v.get("GroupInstanceList")?.as_array()) + .inspect(|arr| { + group_paths.extend( + arr.iter() + .filter_map(|item| { + item.as_object() + .and_then(|v| v.get("GroupPath")?.as_str()) + .map(|group_path| group_path.to_string()) + }) + .collect::>(), + ); + }); + } + + group_paths.par_iter().for_each(|path| { + parse_and_count!( + path, + "RPG.GameCore.RtLevelGroupInfoBase", + assets, + types, + out_folder + ) + }); + + Ok(()) +} + +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + _: &ConfigManifest, +) -> Result<()> { + parse_floor(assets, types, out_folder)?; + parse_group(assets, types, out_folder) +} diff --git a/crates/program/src/actions/config/mission.rs b/crates/program/src/actions/config/mission.rs new file mode 100644 index 0000000..759ac35 --- /dev/null +++ b/crates/program/src/actions/config/mission.rs @@ -0,0 +1,129 @@ +use super::{ConfigManifest, parse_config}; +use crate::{COUNTER_CONFIGS, parse_and_count}; +use anyhow::Result; +use dashmap::DashSet; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde_json::Value; +use std::{ + collections::{HashMap, HashSet}, + fs, + path::Path, + sync::atomic::Ordering, +}; +use tg_parser::DataDefine; + +fn read_performance(base_path: &Path, name: &str, out: &mut HashSet) -> Result<()> { + let entries = + serde_json::from_slice::>(&fs::read(base_path.join(format!("{name}.json")))?)?; + + for item in entries { + let Some(Value::String(performance_path)) = + item.get("PerformancePath").or_else(|| item.get("ActPath")) + else { + continue; + }; + out.insert(performance_path.to_string()); + } + + Ok(()) +} + +/// RPG.GameCore.LevelGraphInfo +fn parse_performances( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, +) { + let mut performances = HashSet::new(); + let base_path = out_folder.join("ExcelOutput"); + + let _ = read_performance(&base_path, "PerformanceA", &mut performances); + let _ = read_performance(&base_path, "PerformanceC", &mut performances); + let _ = read_performance(&base_path, "PerformanceCG", &mut performances); + let _ = read_performance(&base_path, "PerformanceD", &mut performances); + let _ = read_performance(&base_path, "PerformanceDS", &mut performances); + let _ = read_performance(&base_path, "PerformanceE", &mut performances); + let _ = read_performance(&base_path, "PerformanceVideo", &mut performances); + let _ = read_performance(&base_path, "DialogueNPC", &mut performances); + + performances.par_iter().for_each(|path| { + parse_and_count!( + path, + "RPG.GameCore.LevelGraphConfig", + assets, + types, + out_folder + ) + }); +} + +/// RPG.GameCore.MainMissionInfoConfig +fn parse_mission_info( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, +) -> Result<()> { + let entries = serde_json::from_slice::>(&fs::read( + out_folder.join("ExcelOutput/MainMission.json"), + )?)?; + + let paths = entries + .iter() + .filter_map(|item| { + if let Some(Value::Number(mission_id)) = item.get("MainMissionID") { + Some(format!( + "Config/Level/Mission/{mission_id}/MissionInfo_{mission_id}.json" + )) + } else { + None + } + }) + .collect::>(); + + let sub_mission_paths = DashSet::new(); + + paths.par_iter().for_each(|path| { + if let Ok(config) = parse_config( + path, + "RPG.GameCore.MainMissionInfoConfig", + assets, + types, + out_folder, + ) { + COUNTER_CONFIGS.fetch_add(1, Ordering::Relaxed); + if let Some(Value::Array(sub_mission_list)) = config.get("SubMissionList") { + for sub_mission in sub_mission_list { + let Some(Value::String(json_path)) = sub_mission.get("MissionJsonPath") else { + continue; + }; + + sub_mission_paths.insert(json_path.to_string()); + } + } + } + }); + + sub_mission_paths.par_iter().for_each(|path| { + parse_and_count!( + &path, + "RPG.GameCore.LevelGraphConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} + +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + _: &ConfigManifest, +) -> Result<()> { + let _ = parse_mission_info(assets, types, out_folder); + parse_performances(assets, types, out_folder); + + Ok(()) +} diff --git a/crates/program/src/actions/config/mod.rs b/crates/program/src/actions/config/mod.rs new file mode 100644 index 0000000..8a2fc0c --- /dev/null +++ b/crates/program/src/actions/config/mod.rs @@ -0,0 +1,161 @@ +use anyhow::{Context, Result}; +use common::hash; +use rayon::iter::{IntoParallelRefIterator, ParallelBridge as _, ParallelIterator}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::{ + collections::HashMap, + fs, panic, + path::{Path, PathBuf}, +}; +use tg_parser::{DataDefine, DynamicParser, ValueKind}; + +mod adventure_ability; +mod adventure_modifier; +mod complex_skill_ai_global; +mod config_ability; +mod global_modifier; +mod global_task_template; +mod level_output; +mod mission; +mod rogue_chest_map; +mod rogue_npc; +mod skill_tree_point_preset; +mod summon_unit; +mod video_caption; + +#[macro_export] +macro_rules! parse_and_count { + ($path:expr, $type:expr, $assets:expr, $types:expr, $out_folder:expr) => { + if $crate::actions::config::parse_config($path, $type, $assets, $types, $out_folder).is_ok() + { + $crate::COUNTER_CONFIGS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + }; +} + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct ConfigManifest { + adventure_ability_config: Vec, + turn_based_ability_config: Vec, + // battle_lineup_config: Vec, + // battle_lineup_avatar_config: Vec, + // battle_lineup_maze_buff_config: Vec, + battle_lineup_skill_tree_preset_config: Vec, + // #[serde(rename = "BattleLineupCEPresetConfig")] + // battle_lineup_cepreset_config: Vec, + global_modifier_config: Vec, + adventure_modifier_config: Vec, + #[serde(rename = "ComplexSkillAIGlobalGroupConfig")] + complex_skill_aiglobal_group_config: Vec, + global_task_template: Vec, + // common_skill_pool_config: Vec, +} + +#[inline] +fn split_path(path: &str) -> Option<(String, String)> { + path.rsplit_once('/').map(|(dir, file)| { + ( + if dir.is_empty() { "/" } else { dir }.to_string(), + file.to_string(), + ) + }) +} + +fn parse_config( + json_path: &str, + type_name: &str, + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, +) -> Result { + let (folder_path, file_name) = split_path(json_path).context("Invalid path")?; + let path_hash = hash::get_32bit_hash_const(&format!( + "BakedConfig/{}", + json_path.replace(".json", ".bytes") + )); + + let bytes = assets.get(&path_hash).ok_or_else(|| { + // tracing::debug!("Asset not found: {json_path} ({path_hash})"); + anyhow::anyhow!("Asset not found") + })?; + + match panic::catch_unwind(|| { + let mut parser = DynamicParser::new(types, bytes); + parser.parse(&ValueKind::Class(type_name.to_string()), false) + }) { + Ok(Ok(parsed)) => { + let out_folder = out_folder.join(folder_path); + fs::create_dir_all(&out_folder)?; + let out_path = out_folder.join(file_name); + fs::write(&out_path, serde_json::to_string_pretty(&parsed)?) + .context(format!("Failed to write to {out_path:?}"))?; + return Ok(parsed); + } + Ok(Err(err)) => tracing::error!("Parse error for {json_path} ({type_name}): {err:?}"), + Err(err) => tracing::error!("Panic during parsing {json_path} ({type_name}): {err:?}"), + } + + Ok(json!({})) +} + +type ParseFn = + fn(&HashMap>, &HashMap, &Path, &ConfigManifest) -> Result<()>; + +pub fn parse_configs( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + additional_paths: Option, +) -> Result<()> { + tracing::info!("Parsing Configs..."); + let config_manifest_bytes = assets + .get(&hash::get_32bit_hash_const( + "BakedConfig/ConfigManifest.json", + )) + .ok_or_else(|| anyhow::anyhow!("ConfigManifest.json not found"))?; + let config_manifest: ConfigManifest = + serde_json::from_slice(config_manifest_bytes).context("Failed to parse ConfigManifest")?; + + let parses: &[(ParseFn, &str)] = &[ + (adventure_ability::parse, "adventure_ability"), + (config_ability::parse, "config_ability"), + (global_modifier::parse, "global_modifier"), + (skill_tree_point_preset::parse, "skill_tree_point_preset"), + (adventure_modifier::parse, "adventure_modifier"), + (complex_skill_ai_global::parse, "complex_skill_ai_global"), + (global_task_template::parse, "global_task_template"), + (level_output::parse, "level_output"), + (summon_unit::parse, "summon_unit"), + (mission::parse, "mission"), + (video_caption::parse, "video_caption"), + (rogue_npc::parse, "rogue_npc"), + (rogue_chest_map::parse, "rogue_chest_map"), + ]; + + parses.par_iter().for_each(|(parse_fn, name)| { + if let Err(err) = + panic::catch_unwind(|| parse_fn(assets, types, out_folder, &config_manifest).unwrap()) + { + tracing::error!("Failed to parse {}: {:?}", name, err); + } + }); + + // Parse additional paths if provided + additional_paths + .and_then(|path| { + std::fs::read(path).ok().and_then(|bytes| { + serde_json::from_slice::>>(&bytes).ok() + }) + }) + .unwrap_or_default() + .iter() + .flat_map(|(data_type, paths)| paths.iter().map(move |json_path| (json_path, data_type))) + .par_bridge() + .for_each(|(json_path, data_type)| { + parse_and_count!(json_path, data_type, assets, types, out_folder); + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/rogue_chest_map.rs b/crates/program/src/actions/config/rogue_chest_map.rs new file mode 100644 index 0000000..2b8e250 --- /dev/null +++ b/crates/program/src/actions/config/rogue_chest_map.rs @@ -0,0 +1,35 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde_json::Value; +use std::{collections::HashMap, fs, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.RogueChestMapConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + _: &ConfigManifest, +) -> Result<()> { + let summon_unit: Vec = serde_json::from_slice(&fs::read( + out_folder.join("ExcelOutput/RogueDLCChessBoard.json"), + )?)?; + + summon_unit.par_iter().for_each(|summon_unit| { + parse_and_count!( + summon_unit + .get("ChessBoardConfiguration") + .unwrap() + .as_str() + .unwrap(), + "RPG.GameCore.RogueChestMapConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/rogue_npc.rs b/crates/program/src/actions/config/rogue_npc.rs new file mode 100644 index 0000000..f4515ff --- /dev/null +++ b/crates/program/src/actions/config/rogue_npc.rs @@ -0,0 +1,84 @@ +use super::ConfigManifest; +use crate::{COUNTER_CONFIGS, actions::config::parse_config, parse_and_count}; +use anyhow::Result; +use dashmap::DashSet; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde_json::Value; +use std::{collections::HashMap, fs, path::Path, sync::atomic::Ordering}; +use tg_parser::DataDefine; + +#[inline] +fn extract_npc_json_paths>(path: P) -> std::io::Result> { + let data = fs::read(path)?; + let json: Vec = serde_json::from_slice(&data)?; + Ok(json + .into_iter() + .filter_map(|item| item.get("NPCJsonPath")?.as_str().map(|s| s.to_string())) + .collect()) +} + +/// RPG.GameCore.RogueNPCConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + _: &ConfigManifest, +) -> Result<()> { + let paths = [ + "ExcelOutput/RogueNPC.json", + "ExcelOutput/RogueTournNPC.json", + "ExcelOutput/RogueMagicNPC.json", + ] + .iter() + .filter_map(|path| extract_npc_json_paths(out_folder.join(path)).ok()) + .flatten() + .collect::>(); + + let dialogue_paths = DashSet::new(); + let option_paths = DashSet::new(); + + paths.par_iter().for_each(|path| { + if let Ok(config) = parse_config( + path, + "RPG.GameCore.RogueNPCConfig", + assets, + types, + out_folder, + ) { + COUNTER_CONFIGS.fetch_add(1, Ordering::Relaxed); + if let Some(Value::Array(dialogue_list)) = config.get("DialogueList") { + for dialogue in dialogue_list { + if let Some(Value::String(json_path)) = dialogue.get("DialoguePath") { + dialogue_paths.insert(json_path.to_string()); + }; + + if let Some(Value::String(json_path)) = dialogue.get("OptionPath") { + option_paths.insert(json_path.to_string()); + }; + } + } + } + }); + + dialogue_paths.par_iter().for_each(|path| { + parse_and_count!( + &path, + "RPG.GameCore.LevelGraphConfig", + assets, + types, + out_folder + ) + }); + + option_paths.par_iter().for_each(|path| { + parse_and_count!( + &path, + "RPG.GameCore.RogueDialogueEventConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/skill_tree_point_preset.rs b/crates/program/src/actions/config/skill_tree_point_preset.rs new file mode 100644 index 0000000..885734f --- /dev/null +++ b/crates/program/src/actions/config/skill_tree_point_preset.rs @@ -0,0 +1,29 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::{collections::HashMap, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.SkillTreePointPresetConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + config_manifest: &ConfigManifest, +) -> Result<()> { + config_manifest + .battle_lineup_skill_tree_preset_config + .par_iter() + .for_each(|json_path| { + parse_and_count!( + json_path, + "RPG.GameCore.SkillTreePointPresetConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/summon_unit.rs b/crates/program/src/actions/config/summon_unit.rs new file mode 100644 index 0000000..c932284 --- /dev/null +++ b/crates/program/src/actions/config/summon_unit.rs @@ -0,0 +1,31 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde_json::Value; +use std::{collections::HashMap, fs, path::Path}; +use tg_parser::DataDefine; + +/// RPG.GameCore.SummonUnitConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + _: &ConfigManifest, +) -> Result<()> { + let summon_unit: Vec = serde_json::from_slice(&fs::read( + out_folder.join("ExcelOutput/SummonUnitData.json"), + )?)?; + + summon_unit.par_iter().for_each(|summon_unit| { + parse_and_count!( + summon_unit.get("JsonPath").unwrap().as_str().unwrap(), + "RPG.GameCore.SummonUnitConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} diff --git a/crates/program/src/actions/config/video_caption.rs b/crates/program/src/actions/config/video_caption.rs new file mode 100644 index 0000000..4955c1b --- /dev/null +++ b/crates/program/src/actions/config/video_caption.rs @@ -0,0 +1,50 @@ +use super::ConfigManifest; +use crate::parse_and_count; +use anyhow::Result; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde_json::Value; +use std::{collections::HashMap, fs, path::Path}; +use tg_parser::DataDefine; + +#[inline] +fn extract_caption_paths>(path: P) -> std::io::Result> { + let data = fs::read(path)?; + let json: Vec = serde_json::from_slice(&data)?; + Ok(json + .into_iter() + .filter_map(|item| item.get("CaptionPath")?.as_str().map(|s| s.to_string())) + .collect()) +} + +/// RPG.GameCore.VideoCaptionConfig +pub fn parse( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + _: &ConfigManifest, +) -> Result<()> { + let paths = [ + "ExcelOutput/VideoConfig.json", + "ExcelOutput/CutSceneConfig.json", + "ExcelOutput/LoopCGConfig.json", + ] + .iter() + .filter_map(|path| extract_caption_paths(out_folder.join(path)).ok()) + .flatten() + .collect::>(); + + paths + .par_iter() + .filter(|path| !path.is_empty()) + .for_each(|path| { + parse_and_count!( + path, + "RPG.GameCore.VideoCaptionConfig", + assets, + types, + out_folder + ) + }); + + Ok(()) +} diff --git a/crates/program/src/actions/excel.rs b/crates/program/src/actions/excel.rs new file mode 100644 index 0000000..e993c0f --- /dev/null +++ b/crates/program/src/actions/excel.rs @@ -0,0 +1,59 @@ +use anyhow::Context; +use common::hash; +use rayon::iter::{IntoParallelRefIterator as _, ParallelIterator as _}; +use std::{collections::HashMap, fs, path::Path, sync::atomic::Ordering}; +use tg_parser::{DataDefine, DynamicParser, ValueKind}; + +use crate::COUNTER_EXCELS; + +pub fn parse_all_excels( + assets: &HashMap>, + types: &HashMap, + out_folder: &Path, + excel_paths: &HashMap>, +) -> anyhow::Result<()> { + tracing::info!("Parsing Excels..."); + + let out_excel = out_folder.join("ExcelOutput"); + + if !out_excel.is_dir() { + fs::create_dir_all(&out_excel).context("Failed create ExcelOutput directory")?; + } + + excel_paths.par_iter().for_each(|(type_name, paths)| { + let kind = ValueKind::Array(Box::new(ValueKind::Class(type_name.to_string()))); + for path in paths { + let Some(bytes) = assets.get(&hash::get_32bit_hash_const(path)) else { + continue; + }; + + // Skip empty first byte + let bytes = if !bytes.is_empty() && bytes[0] == 0 { + &bytes[1..] + } else { + bytes + } + .to_vec(); + + let mut parser = DynamicParser::new(types, &bytes); + match parser.parse(&kind, false) { + Ok(parsed) => { + let file_name = path.split("/").last().unwrap().replace(".bytes", ".json"); + let file_out = if file_name.starts_with("Textmap") { + continue; + } else { + out_excel.join(file_name) + }; + fs::write(file_out, serde_json::to_string_pretty(&parsed).unwrap()).unwrap(); + COUNTER_EXCELS.fetch_add(1, Ordering::Relaxed) + } + Err(err) => { + tracing::error!("failed to parse {:?} {} {err}", kind, path); + continue; + } + }; + } + }); + + Ok(()) +} diff --git a/crates/program/src/actions/mod.rs b/crates/program/src/actions/mod.rs new file mode 100644 index 0000000..27b2db1 --- /dev/null +++ b/crates/program/src/actions/mod.rs @@ -0,0 +1,3 @@ +pub mod config; +pub mod excel; +pub mod textmap; diff --git a/crates/program/src/actions/textmap.rs b/crates/program/src/actions/textmap.rs new file mode 100644 index 0000000..8ea5f95 --- /dev/null +++ b/crates/program/src/actions/textmap.rs @@ -0,0 +1,230 @@ +use std::{collections::HashMap, io::Cursor, path::Path, sync::atomic::Ordering}; + +use anyhow::Result; +use common::hash::get_32bit_hash_const; +use serde::Serialize; +use serde_json::{Map, Value}; +use tg_bytes_util::{ExistFlag, FromBytes}; + +use crate::COUNTER_TEXTMAPS; + +pub const TEXTMAP_PATHS: [(&str, i32); 28] = [ + ( + "TextMapEN.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_en.bytes"), + ), + ( + "TextMapCN.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_cn.bytes"), + ), + ( + "TextMapKR.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_kr.bytes"), + ), + ( + "TextMapJP.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_jp.bytes"), + ), + ( + "TextMapID.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_id.bytes"), + ), + ( + "TextMapCHS.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_chs.bytes"), + ), + ( + "TextMapCHT.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_cht.bytes"), + ), + ( + "TextMapDE.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_de.bytes"), + ), + ( + "TextMapES.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_es.bytes"), + ), + ( + "TextMapFR.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_fr.bytes"), + ), + ( + "TextMapRU.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_ru.bytes"), + ), + ( + "TextMapTH.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_th.bytes"), + ), + ( + "TextMapVI.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_vi.bytes"), + ), + ( + "TextMapPT.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/Textmap_pt.bytes"), + ), + ( + "TextMapMainEN.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_en.bytes"), + ), + ( + "TextMapMainCN.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_cn.bytes"), + ), + ( + "TextMapMainKR.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_kr.bytes"), + ), + ( + "TextMapMainJP.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_jp.bytes"), + ), + ( + "TextMapMainID.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_id.bytes"), + ), + ( + "TextMapMainCHS.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_chs.bytes"), + ), + ( + "TextMapMainCHT.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_cht.bytes"), + ), + ( + "TextMapMainDE.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_de.bytes"), + ), + ( + "TextMapMainES.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_es.bytes"), + ), + ( + "TextMapMainFR.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_fr.bytes"), + ), + ( + "TextMapMainRU.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_ru.bytes"), + ), + ( + "TextMapMainTH.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_th.bytes"), + ), + ( + "TextMapMainVI.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_vi.bytes"), + ), + ( + "TextMapMainPT.json", + get_32bit_hash_const("BakedConfig/ExcelOutput/TextmapMain_pt.bytes"), + ), +]; + +#[derive(Serialize)] +#[serde(rename_all = "PascalCase")] +struct TextID { + pub hash: i32, + pub hash_64: u64, +} + +impl FromBytes for TextID { + fn from_bytes(r: &mut T) -> std::io::Result { + Ok(Self { + hash: i32::from_bytes(r)?, + hash_64: u64::from_bytes(r)?, + }) + } +} + +#[derive(Serialize)] +#[serde(rename_all = "PascalCase")] +struct TextMapRow { + #[serde(rename = "ID")] + pub id: Option, + pub text: String, + pub has_param: bool, +} + +impl FromBytes for TextMapRow { + fn from_bytes(r: &mut T) -> std::io::Result { + let exist_flag = ExistFlag::new(r, 3)?; + Ok(Self { + id: if exist_flag.exists(0) { + Some(TextID::from_bytes(r)?) + } else { + None + }, + text: if exist_flag.exists(1) { + String::from_bytes(r)? + } else { + String::with_capacity(0) + }, + has_param: if exist_flag.exists(2) { + bool::from_bytes(r)? + } else { + false + }, + }) + } +} + +pub fn parse_all_textmap( + assets: &HashMap>, + out_folder: &Path, + minimal: bool, +) -> Result<()> { + tracing::info!("Parsing Textmaps..."); + + let out_folder = out_folder.join("TextMap"); + + if !out_folder.exists() { + std::fs::create_dir_all(&out_folder)?; + } + + for (name, hash) in TEXTMAP_PATHS { + let Some(asset) = assets.get(&hash) else { + continue; + }; + + // Skip empty first byte + let asset = if !asset.is_empty() && asset[0] == 0 { + &asset[1..] + } else { + asset + }; + + let out_path = out_folder.join(name); + let mut cursor = Cursor::new(asset); + + let Ok(parsed) = Vec::::from_bytes(&mut cursor) else { + continue; + }; + + if minimal { + std::fs::write( + out_path, + serde_json::to_string_pretty( + &parsed + .into_iter() + .map(|row| { + ( + row.id.map(|v| v.hash).unwrap_or_default().to_string(), + Value::String(row.text), + ) + }) + .collect::>(), + ) + .unwrap(), + ) + .unwrap(); + COUNTER_TEXTMAPS.fetch_add(1, Ordering::Relaxed); + } else { + std::fs::write(out_path, serde_json::to_string_pretty(&parsed).unwrap()).unwrap(); + } + } + + Ok(()) +} diff --git a/crates/program/src/cli.rs b/crates/program/src/cli.rs new file mode 100644 index 0000000..ede0567 --- /dev/null +++ b/crates/program/src/cli.rs @@ -0,0 +1,67 @@ +use std::path::PathBuf; + +use clap::{Args, Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "tg-parser", + version = "1.0", + about = "Tool for parsing certain anime game resources." +)] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand)] +pub enum Command { + /// Process textmap only + Textmap { + /// Persistent path or design data URL + input_url: String, + /// Output directory + output_dir: PathBuf, + + /// Parse full textmap structure as array, rather than just key-value pair + #[arg(long)] + full_textmap: bool, + + /// Save .bytes file after downloading the files + #[arg(long, name = "save-bytes-file")] + save_bytes_file: bool, + }, + + /// Process excel only + Excels(ExcelArgs), + + /// Process excel, config, textmap parse + All(ExcelArgs), +} + +#[derive(Args)] +pub struct ExcelArgs { + /// data.json schema file path + pub data_json: String, + /// excel_paths.json file path + pub excel_path_json: String, + /// Persistent path or design data URL + pub input_url: String, + /// Output directory + pub output_dir: PathBuf, + + /// Parse full textmap structure as array, rather than just key-value pair + #[arg(long)] + pub full_textmap: bool, + + /// Save .bytes file after downloading the files + #[arg(long, name = "save-bytes-file")] + pub save_bytes_file: bool, + + /// Log all error into console + #[arg(long, name = "log-error")] + pub log_error: bool, + + /// Additional configs path to parse, with type as key, and array of paths as values + #[arg(long, name = "config-paths")] + pub config_paths: Option, +} diff --git a/crates/program/src/main.rs b/crates/program/src/main.rs new file mode 100644 index 0000000..7937ca1 --- /dev/null +++ b/crates/program/src/main.rs @@ -0,0 +1,141 @@ +use crate::{ + actions::textmap::TEXTMAP_PATHS, + cli::{Cli, Command}, +}; +use anyhow::{Context as _, Result}; +use clap::Parser; +use common::downloader; +use std::{ + collections::HashMap, + fs, + sync::atomic::{AtomicI32, Ordering}, + time::Instant, +}; +use tg_parser::DataDefine; +use tracing::Level; + +mod actions; +mod cli; + +pub static COUNTER_CONFIGS: AtomicI32 = AtomicI32::new(0); +pub static COUNTER_EXCELS: AtomicI32 = AtomicI32::new(0); +pub static COUNTER_TEXTMAPS: AtomicI32 = AtomicI32::new(0); + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match &cli.command { + Command::Textmap { + input_url, + output_dir, + full_textmap, + save_bytes_file, + } => { + common::logging::init(Level::INFO); + + let start = Instant::now(); + + let assets = downloader::download_all_design_data( + input_url.clone(), + if *save_bytes_file { + Some(output_dir.clone()) + } else { + None + }, + TEXTMAP_PATHS.iter().map(|v| v.1).collect(), + )?; + + tracing::info!("Download Done! Took {}s", start.elapsed().as_secs()); + + let start = Instant::now(); + + actions::textmap::parse_all_textmap(&assets, output_dir, !full_textmap)?; + + tracing::info!("Textmap Parse Done! Took {}ms", start.elapsed().as_millis()); + } + + Command::Excels(args) | Command::All(args) => { + if args.log_error { + common::logging::init(Level::INFO) + } else { + common::logging::init_info_only(); + } + + let assets = downloader::download_all_design_data( + args.input_url.clone(), + if args.save_bytes_file { + Some(args.output_dir.clone()) + } else { + None + }, + Vec::with_capacity(0), + )?; + + let start = Instant::now(); + + let excel_paths: HashMap> = serde_json::from_slice( + &fs::read(&args.excel_path_json).context("Failed to read excel_paths.json")?, + )?; + + let types: HashMap = serde_json::from_slice( + &fs::read(&args.data_json).context("Failed to read data.json")?, + )?; + + actions::excel::parse_all_excels( + &assets, + &types, + &args.output_dir.clone(), + &excel_paths, + )?; + + if let Command::All(_) = cli.command { + actions::config::parse_configs( + &assets, + &types, + &args.output_dir, + args.config_paths.clone(), + )?; + actions::textmap::parse_all_textmap(&assets, &args.output_dir, !args.full_textmap)?; + } + + tracing::info!( + "Parsed {} Excels, {} Configs, and {} Textmaps in {}s", + COUNTER_EXCELS.load(Ordering::Relaxed), + COUNTER_CONFIGS.load(Ordering::Relaxed), + COUNTER_TEXTMAPS.load(Ordering::Relaxed), + start.elapsed().as_secs() + ); + } + } + + Ok(()) +} + +// fn main() { +// use std::collections::HashMap; +// use tg_parser::{DynamicParser, ValueKind}; +// let assets = common::downloader::download_all_design_data( +// String::from( +// "C:/Data/hoyoreverse/StarRail_3.3.51/StarRail_Data/Persistent/DesignData/Windows", +// ), +// None, +// ) +// .unwrap(); +// common::logging::init(tracing::Level::DEBUG); +// let bytes = assets +// .get(&common::hash::get_32bit_hash( +// "BakedConfig/Config/AudioConfig.bytes", +// )) +// .unwrap(); +// let bytes = &bytes[12..].to_vec(); +// let schema: HashMap = +// serde_json::from_slice(&std::fs::read("data.json").unwrap()).unwrap(); +// let mut parser = DynamicParser::new(&schema, &bytes); +// let parsed = parser +// .parse( +// &ValueKind::Class(String::from("RPG.GameCore.AudioConfig")), +// false, +// ) +// .unwrap(); +// std::fs::write("ss.json", serde_json::to_string_pretty(&parsed).unwrap()).unwrap(); +// }