diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/01_process_dump_data.py b/01_process_dump_data.py new file mode 100644 index 0000000..3f554dc --- /dev/null +++ b/01_process_dump_data.py @@ -0,0 +1,71 @@ +import sys +import os +import shutil +import json +import py7zr +import yaml +from tqdm import tqdm +import subprocess + +from pathlib import Path + +with open('config/config.yaml', 'r') as fin: + config = yaml.load(fin, Loader=yaml.FullLoader) + +exe_7z = config["tools"]["7z_exe"] + +src = Path(config["archives"]["source_folder"]) +dst = Path(config["archives"]["target_folder"]) + +archive_complete = Path(config["archives"]["backup_file"]) + +used = { + dst / config["archives"]["loot_filename"]: lambda x: x.startswith("resp.client.location.getLocalloot"), + dst / config["archives"]["bot_filename"]: lambda x: x.startswith("resp.client.game.bot.generate"), +} + +input_files = set(os.listdir(src)) + +for archive_dst, filt in used.items(): + + mode = None + if os.path.exists(archive_dst): + mode = 'a' + with py7zr.SevenZipFile(archive_dst, 'r') as archive: + archive_files = set(archive.getnames()) + else: + mode = 'w' + archive_files = [] + + # filter out every file which is already in the archive + files = input_files.difference(archive_files) + + # get only those files which fit the recycle filters + print(f"Number of files which are not found in the archive: {len(files)}") + files = [ + fi for fi in files + if filt(fi) + ] + print(f"Of those according to filter add {len(files)} files to {archive_dst}") + + # temporary copy to destination folder to add them via 7z subprocess + if len(files) > 0: + for file in tqdm(files): + shutil.copy(src / file, dst / file) + + cmd = f"{exe_7z} a -t7z {archive_dst} *.json" + os.chdir(dst) + subprocess.call(cmd) + + for file in tqdm(files): + os.remove(dst / file) + +# add all files to backup archive (if they are not inside already) +cmd = f"{exe_7z} a -up1q1r2x1y1z1w1 -t7z {archive_complete} *.json" +os.chdir(src) +subprocess.call(cmd) + +# delete raw data from source folder +files = os.listdir(src) +for file in tqdm(files): + os.remove(src / file) diff --git a/02_loot_generator_parallel.py b/02_loot_generator_parallel.py new file mode 100644 index 0000000..9f84f1c --- /dev/null +++ b/02_loot_generator_parallel.py @@ -0,0 +1,219 @@ +import json +import yaml +import time +import py7zr +import hashlib +import datetime + +from tqdm import tqdm +from pathlib import Path + +from concurrent import futures + +from collections import defaultdict +from itertools import groupby + +from src.tarkov_items import TarkovItems + +from src.static_loot import preprocess_staticloot, StaticLootProcessor +from src.loose_loot import preprocess_looseloot, LooseLootProcessor + +with open('config/config.yaml', 'r') as fin: + config = yaml.load(fin, Loader=yaml.FullLoader) + +with open(f'config/{config["config"]["static"]["forced_static_yaml"]}', 'r') as fin: + FORCED_STATIC = yaml.load(fin, Loader=yaml.FullLoader) + +with open(f'config/{config["config"]["static"]["forced_loose_yaml"]}', 'r') as fin: + FORCED_LOOSE = yaml.load(fin, Loader=yaml.FullLoader) + +with open(f'config/{config["server"]["map_directory_mapping_yaml"]}', 'r') as fin: + loose_loot_dir_map = yaml.load(fin, Loader=yaml.FullLoader) + +STATIC_WEAPON_IDS = config["config"]["static"]["static_weapon_ids"] + +tarkov_server_dir = Path(config["server"]["location"]) +loot_dump_archive = Path(config["archives"]["target_folder"]) / config["archives"]["loot_filename"] + + +def hash_file(text): + sha256 = hashlib.sha256() + sha256.update(text) + return sha256.hexdigest() + + +def parse_dumps(input): + fname = input[0] + bio = input[1] + text = bio.read() + + fi = json.loads(text) + + datestr = fname.split(".getLocalloot_")[-1].split(".")[0] + date = datetime.datetime.strptime(datestr, "%Y-%m-%d_%H-%M-%S") + + if fi["data"] is not None: + basic_info = { + "map": fi["data"]["Name"], + "filehash": hash_file(text), + "date": date, + "fname": fname + } + + looseloot = [li for li in fi["data"]["Loot"] if not li["IsStatic"]] + staticloot = [li for li in fi["data"]["Loot"] if li["IsStatic"]] + + looseloot_processed = preprocess_looseloot(looseloot) + containers = preprocess_staticloot(staticloot, STATIC_WEAPON_IDS) + + return { + "basic_info": basic_info, + "looseloot": looseloot_processed, + "containers": containers + } + else: + return None + + +def main(): + tarkov_items = TarkovItems( + items=tarkov_server_dir / "project/assets/database/templates/items.json", + handbook=tarkov_server_dir / "project/assets/database/templates/handbook.json", + locales=tarkov_server_dir / "project/assets/database/locales/global/en.json" + ) + + loose_loot_dir = tarkov_server_dir / "project/assets/database/locations" + static_loot_dir = tarkov_server_dir / "project/assets/database/loot" + + print("Open dump archive", end="; ") + map_files = {} + gather_loot_results = [] + time_start = time.time() + with py7zr.SevenZipFile(loot_dump_archive, 'r') as archive: + archive_files = set(archive.getnames()) + with futures.ProcessPoolExecutor() as executor: + print("Gathering dumps") + for result in list(tqdm(executor.map(parse_dumps, archive.read(archive_files).items()), total=len(archive_files))): + if result is not None: + gather_loot_results.append(result) + # get the newest dump per map + mapi = result["basic_info"]["map"] + if mapi not in map_files: + map_files[mapi] = ( + result["basic_info"]["date"], + result["basic_info"]["fname"] + ) + else: + if result["basic_info"]["date"] > map_files[mapi][0]: + map_files[mapi] = ( + result["basic_info"]["date"], + result["basic_info"]["fname"] + ) + + + print(f"Reading dumps took {time.time() - time_start} seconds.") + dump_count = len(gather_loot_results) + + # remove duplicate dumps + time_start = time.time() + gather_loot_results = sorted(gather_loot_results, key=lambda x: x["basic_info"]["filehash"]) + gather_loot_results_unique = [] + for _, g in groupby(gather_loot_results, key=lambda x: x["basic_info"]["filehash"]): + g = list(g) + if len(g) > 1: + # print(f"Duplicate dumps: {', '.join([gi['filename'] for gi in g])}") + pass + gather_loot_results_unique.append(g[0]) + + del gather_loot_results + dump_count_unique = len(gather_loot_results_unique) + + print( + f"Removing duplicates took {time.time() - time_start} seconds: {dump_count - dump_count_unique} / {dump_count}") + + # Map Reduce + print("Map reducing dumps", end="; ") + time_start = time.time() + gather_loot_results_unique = sorted(gather_loot_results_unique, key=lambda x: x["basic_info"]["map"]) + looseloot_counts = {} + container_counts = [] + map_counts = {} + for mapi, g in groupby(gather_loot_results_unique, key=lambda x: x["basic_info"]["map"]): + g = list(g) + map_counts[mapi] = len(g) + looseloot_counts[mapi] = {} + + looseloot_counts[mapi]["counts"] = defaultdict(int) + looseloot_counts[mapi]["items"] = defaultdict(list) + looseloot_counts[mapi]["itemproperties"] = defaultdict(list) + looseloot_counts[mapi]["map_spawnpoint_count"] = [] + + for gi in g: + + container_counts += gi["containers"] + + for k, v in gi["looseloot"]["counts"].items(): + looseloot_counts[mapi]["counts"][k] += v + for k, v in gi["looseloot"]["items"].items(): + looseloot_counts[mapi]["items"][k] += v + for k, v in gi["looseloot"]["itemproperties"].items(): + looseloot_counts[mapi]["itemproperties"][k] += v + + looseloot_counts[mapi]["map_spawnpoint_count"] += [gi["looseloot"]["map_spawnpoint_count"]] + + del gather_loot_results_unique + print(f"took {time.time() - time_start} seconds.") + + static_loot_processor = StaticLootProcessor( + tarkov_items=tarkov_items, + static_weapon_ids=STATIC_WEAPON_IDS, + forced_static=FORCED_STATIC + ) + # create static containers (containers per map, forced loot in map, static weapons in map) + print("Create \"static containers\"", end='; ') + time_start = time.time() + static_containers = {} + with py7zr.SevenZipFile(loot_dump_archive, 'r') as archive: + targets = [datename_tuple[1] for _, datename_tuple in map_files.items()] + targets = sorted(targets) + for fname, bio in archive.read(targets).items(): + mapi, static_containers_mi = static_loot_processor.create_static_containers(bio) + static_containers[mapi] = static_containers_mi + print(f"took {time.time() - time_start} seconds.") + with open(static_loot_dir / "staticContainers.json", "w") as fout: + json.dump(static_containers, fout, indent=1) + + # Ammo distribution + time_start = time.time() + print(f"Creating \"ammo\" distribution", end="; ") + ammo_distribution = static_loot_processor.create_ammo_distribution(container_counts) + print(f"took {time.time() - time_start} seconds.") + with open(static_loot_dir / "staticAmmo.json", "w") as fout: + json.dump(ammo_distribution, fout, indent=1) + + # Static loot distribution + time_start = time.time() + print(f"Creating \"static container\"", end='; ') + static_loot_distribution = static_loot_processor.create_static_loot_distribution(container_counts) + print(f"took {time.time() - time_start} seconds.") + with open(static_loot_dir / "staticLoot.json", 'w') as fout: + json.dump(static_loot_distribution, fout, indent=1) + + # Loose loot distribution + loose_loot_processor = LooseLootProcessor( + tarkov_items=tarkov_items, + FORCED_LOOSE=FORCED_LOOSE + ) + time_start = time.time() + print(f"Calculating \"loose loot\" distribution", end='; ') + loose_loot_distribution = loose_loot_processor.create_loose_loot_distribution(map_counts, looseloot_counts) + print(f"took {time.time() - time_start} seconds") + + for mi, cnt in map_counts.items(): + for mapdir in loose_loot_dir_map[mi]: + with open(loose_loot_dir / mapdir / "looseLoot.json", "w") as fout: + json.dump(loose_loot_distribution[mi], fout, indent=1) + + +if __name__ == '__main__': + main() diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..e124b1e --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,30 @@ +--- +tools: + 7z_exe: 'C:/Program Files/7-Zip/7z.exe' +server: + # SPT-AKI Server location (to store the generated json) + location: 'C:/GAMES/SPT_AKI_Dev/Server/' + # The map names in the loot dump do not directly corresond to the server database structure, this is the mapping + map_directory_mapping_yaml: map_directory_mapping.yaml + +archives: + # where we put our sequential dump jsons to be sorted archived, preprocessor will put them into archives + source_folder: 'D:/Games/SPT_Dev/map_dumps/12.12/' + # here we store the archives with used data + target_folder: 'D:/Games/SPT_Dev/map_dumps/12.12_used/' + # archive name for map loot dump data + loot_filename: loot_dumps.7z + # archive name for bot dump data + bot_filename: bot_dumps.7z + # all dump data is appended into an archive in this folder + backup_file: 'D:/Games/SPT_Dev/map_dumps/12.12_unused/complete.7z' +config: + static: + # ids for static weapons + static_weapon_ids: + - 5d52cc5ba4b9367408500062 + - 5cdeb229d7f00c000e7ce174 + # we have a separate yaml that defined forced static loot per map + forced_static_yaml: forced_static.yaml + # known quest items per map for validation + forced_loose_yaml: forced_loose.yaml \ No newline at end of file diff --git a/config/forced_loose.yaml b/config/forced_loose.yaml new file mode 100644 index 0000000..9195040 --- /dev/null +++ b/config/forced_loose.yaml @@ -0,0 +1,72 @@ +--- +Customs: +- itemTpl: 5938188786f77474f723e87f # Case 0031 +- itemTpl: 5c12301c86f77419522ba7e4 # Flash drive with fake info +- itemTpl: 593965cf86f774087a77e1b6 # Case 0048 +- itemTpl: 591092ef86f7747bb8703422 # Secure folder 0022 in big red offices +- itemTpl: 590c62a386f77412b0130255 # Sliderkey Secure Flash drive in Dorms 2-way room 220 +- itemTpl: 5939e9b286f77462a709572c # Sealed letter (Terragroup) +- itemTpl: 5ac620eb86f7743a8e6e0da0 # Package of graphics cards in big red offices +- itemTpl: 590dde5786f77405e71908b2 # Bank case +- itemTpl: 5910922b86f7747d96753483 # Carbon case +- itemTpl: 5937fd0086f7742bf33fc198 # Bronze pocket watch on a chain +- itemTpl: 5939a00786f7742fe8132936 # Golden Zibbo lighter +- itemTpl: 5939e5a786f77461f11c0098 # Secure Folder 0013 + +Woods: +- itemTpl: 5938878586f7741b797c562f # Case 0052 +- itemTpl: 5d3ec50586f774183a607442 # Jaeger's message Underneath the wooden lookout post. +- itemTpl: 5af04e0a86f7743a532b79e2 # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope +- itemTpl: 5a687e7886f7740c4a5133fb # Blood sample +- itemTpl: 5af04c0b86f774138708f78e # Motor Controller: item_barter_electr_controller + +Shoreline: +- itemTpl: 5a294d7c86f7740651337cf9 # Drone 1 SAS disk +- itemTpl: 5a294d8486f774068638cd93 # Drone 2 SAS disk: ambiguous with itemTpl 5a294f1686f774340c7b7e4a +- itemTpl: 5efdafc1e70b5e33f86de058 # Sanitar's Surgery kit marked with a blue symbol +- itemTpl: 5939e5a786f77461f11c0098 # Secure folder 0013 +- itemTpl: 5a6860d886f77411cd3a9e47 # Secure folder 0060 +- itemTpl: 5a29357286f77409c705e025 # Sliderkey Flash drive +- itemTpl: 5efdaf6de6a30218ed211a48 # Sanitar's Ophthalmoscope In potted plant on dining room table. +- itemTpl: 5d357d6b86f7745b606e3508 # Photo album in west wing room 303 +- itemTpl: 5b4c72b386f7745b453af9c0 # Motor Controller: item_barter_electr_controller2 +- itemTpl: 5a0448bc86f774736f14efa8 # Key to the closed premises of the Health Resort +- itemTpl: 5a29276886f77435ed1b117c # Working hard drive +- itemTpl: 5b4c72fb86f7745cef1cffc5 # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope2 +- itemTpl: 5b4c72c686f77462ac37e907 # Motor Controller: item_barter_electr_controller3 +- itemTpl: 5b43237186f7742f3a4ab252 # Chemical container: item_quest_chem_container +- itemTpl: 5a29284f86f77463ef3db363 # Toughbook reinforced laptop + + +Interchange: +- itemTpl: 5ae9a18586f7746e381e16a3 # OLI cargo manifests +- itemTpl: 5ae9a0dd86f7742e5f454a05 # Goshan cargo manifests +- itemTpl: 5ae9a1b886f77404c8537c62 # Idea cargo manifests +- itemTpl: 5ae9a25386f7746dd946e6d9 # OLI cargo route documents (locked) +- itemTpl: 5ae9a3f586f7740aab00e4e6 # Clothes design handbook - Part 1 +- itemTpl: 5ae9a4fc86f7746e381e1753 # Clothes design handbook - Part 2 +- itemTpl: 5b4c81a086f77417d26be63f # Chemical container item_quest_chem_container2 +- itemTpl: 5b4c81bd86f77418a75ae159 # Chemical container item_quest_chem_container3 + +Factory: +- itemTpl: 591093bb86f7747caa7bb2ee # On the neck of the dead scav in the bunker +- itemTpl: 593a87af86f774122f54a951 # Syringe with a chemical + +Lighthouse: +- itemTpl: 61904c9df62c89219a56e034 # The message is tucked under the bottom of the door to the cabin. +- itemTpl: 619268ad78f4fa33f173dbe5 # Water pump operation data On the desk between other documents in the upper office. +- itemTpl: 619268de2be33f2604340159 # Pumping Station Operation Data In the upper floor office on the shelf. +- itemTpl: 61a00bcb177fb945751bbe6a # Stolen military documents On the back corner of the dining room table on the third floor at Chalet. +- itemTpl: 619252352be33f26043400a7 # Laptop with information + +ReserveBase: +- itemTpl: 60915994c49cf53e4772cc38 # Military documents 1 on the table inside bunker control room +- itemTpl: 60a3b6359c427533db36cf84 # Military documents 2 On the bottom shelf of the cupboard near the corner. +- itemTpl: 60a3b65c27adf161da7b6e14 # Military documents 3 Inside the cupboard next to the 4x4 Weapon Box. +- itemTpl: 608c22a003292f4ba43f8a1a # Medical record 1 (locked by RB-KSM key) +- itemTpl: 60a3b5b05f84d429b732e934 # Medical record 2 (locked by RB-SMP key) +- itemTpl: 609267a2bb3f46069c3e6c7d # T-90M Commander Control Panel +- itemTpl: 60c080eb991ac167ad1c3ad4 # MBT Integrated Navigation System + +Laboratory: +- itemTpl: 5eff135be0d3331e9d282b7b # Flash drive marked with blue tape \ No newline at end of file diff --git a/config/forced_static.yaml b/config/forced_static.yaml new file mode 100644 index 0000000..782386a --- /dev/null +++ b/config/forced_static.yaml @@ -0,0 +1,5 @@ +--- +Customs: +# unknown key +- containerId: custom_multiScene_00058 + itemTpl: 593962ca86f774068014d9af \ No newline at end of file diff --git a/config/map_directory_mapping.yaml b/config/map_directory_mapping.yaml new file mode 100644 index 0000000..bf967cd --- /dev/null +++ b/config/map_directory_mapping.yaml @@ -0,0 +1,18 @@ +--- +Customs: +- bigmap +Factory: +- factory4_day +- factory4_night +Interchange: +- interchange +Laboratory: +- laboratory +Lighthouse: +- lighthouse +ReserveBase: +- rezervbase +Shoreline: +- shoreline +Woods: +- woods \ No newline at end of file diff --git a/src/loose_loot.py b/src/loose_loot.py new file mode 100644 index 0000000..bd38810 --- /dev/null +++ b/src/loose_loot.py @@ -0,0 +1,157 @@ +import json +import copy +import numpy as np +from itertools import groupby +from collections import defaultdict + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from src.tarkov_items import TarkovItems + +def preprocess_looseloot(looseloot): + looseloot_ci = {} + looseloot_ci["counts"] = defaultdict(int) + looseloot_ci["items"] = defaultdict(list) + looseloot_ci["itemproperties"] = defaultdict(list) + looseloot_ci["map_spawnpoint_count"] = len(looseloot) + + unique_ids = {} + for li in looseloot: + + group_fun = lambda x: ( + x["Position"]["x"], + x["Position"]["y"], + x["Position"]["z"], + x["Rotation"]["x"], + x["Rotation"]["y"], + x["Rotation"]["z"], + x["useGravity"], + x["IsGroupPosition"], + ) + + # the bsg ids are insane. + # Sometimes the last 7 digits vary but they spawn the same item at the same position + # e.g. for the quest item "60a3b65c27adf161da7b6e14" at "loot_bunker_quest (3)555192" + # so the first approach was to remove the last digits. + # We then saw, that sometimes when the last digits differ for the same string, also the position + # differs. + # We decided to group over the position/rotation/useGravity since they make out a distinct spot + sane_id = str(group_fun(li)) + if sane_id not in unique_ids: + unique_ids[sane_id] = li["Id"] + looseloot_ci["counts"][sane_id] += 1 + else: + pass + # print(f'Spawn Points dupe: {fname} {unique_ids[sane_id]} = {li["Id"]}') + # continue + + looseloot_ci["items"][sane_id].append(li["Items"][0]["_tpl"]) + looseloot_ci["itemproperties"][sane_id].append(li) + + return looseloot_ci + + +class LooseLootProcessor: + + def __init__(self, tarkov_items: 'TarkovItems', FORCED_LOOSE): + self._tarkov_items = tarkov_items + self._FORCED_LOOSE = FORCED_LOOSE + + def create_loose_loot_distribution(self, map_counts, looseloot_counts): + probabilities = {} + loose_loot_distribution = {} + for mi, map_cnt in map_counts.items(): + probabilities[mi] = {} + for idi, cnt in looseloot_counts[mi]["counts"].items(): + probabilities[mi][idi] = cnt / map_cnt + + loose_loot_distribution[mi] = { + "spawnpointCount": { + "mean": np.mean(looseloot_counts[mi]["map_spawnpoint_count"]), + "std": np.std(looseloot_counts[mi]["map_spawnpoint_count"]) + }, + "spawnpointsForced": [], + "spawnpoints": [] + } + + for spawnpoint, itemlist in looseloot_counts[mi]["itemproperties"].items(): + itemscounts = defaultdict(int) + + for item in itemlist: + itemscounts[item["Items"][0]["_tpl"]] += 1 + + # Group by arguments to create possible positions / rotations per spawnpoint + group_fun = lambda x: ( + x["Position"]["x"], + x["Position"]["y"], + x["Position"]["z"], + x["Rotation"]["x"], + x["Rotation"]["y"], + x["Rotation"]["z"], + x["useGravity"], + x["GroupPositions"], + x["IsGroupPosition"], + ) + + # check if grouping is unique + itemlist_sorted = sorted(itemlist, key=group_fun) + itemlist_grouped = groupby(itemlist_sorted, group_fun) + ks = [] + for k, g in itemlist_grouped: + ks.append(k) + gl = list(g) + if len(ks) > 1: + print(ks) + print(json.dumps(gl, indent=4)) + raise ValueError("Attributes for distinct SpawnPointId differ") + + template = copy.deepcopy(gl[0]) + template["Root"] = None + + item_distribution = [ + { + "tpl": tpl, + "relativeProbability": cnt, + } + for tpl, cnt in itemscounts.items() + ] + + if any((self._tarkov_items.is_questitem(item['tpl']) for item in item_distribution)): + spawnpoint = { + "probability": probabilities[mi][spawnpoint], + "template": template + } + loose_loot_distribution[mi]["spawnpointsForced"].append(spawnpoint) + elif probabilities[mi][spawnpoint] > 0.99: + spawnpoint = { + "probability": probabilities[mi][spawnpoint], + "template": template + } + loose_loot_distribution[mi]["spawnpointsForced"].append(spawnpoint) + print(f"Warning: High probability if spawnpoint {template['Id']} even though no quest item found inside it") + else: + template["Items"] = [] + + spawnpoint = { + "probability": probabilities[mi][spawnpoint], + "template": template, + "itemDistribution": sorted(item_distribution, key=lambda x: x["tpl"]) + } + loose_loot_distribution[mi]["spawnpoints"].append(spawnpoint) + + loose_loot_distribution[mi]["spawnpoints"] = sorted(loose_loot_distribution[mi]["spawnpoints"], key=lambda x: x["template"]["Id"]) + + # Cross check with forced loot in dumps vs items defined in forced_loose.yaml + forced_tpls_file = set([forceditem["itemTpl"] for forceditem in self._FORCED_LOOSE[mi]]) + forced_tpls_found = set([forceditem["template"]["Items"][0]["_tpl"] for forceditem in loose_loot_distribution[mi]["spawnpointsForced"]]) + + # all the tpls that are defined in the forced_loose.yaml for this map that are not found as forced + missing_in_dumps = forced_tpls_file - forced_tpls_found + if len(missing_in_dumps) > 0: + print(f"Error: {mi} Items defined in forced_loose.yaml were not found as quest item:\n {missing_in_dumps}") + # all the tpls that are found as forced but not in the forced_loose.yaml + missing_in_file = forced_tpls_found - forced_tpls_file + if len(missing_in_file) > 0: + print(f"Warning: {mi} Items were found as forced in dump but were not defined in forced_loose.yaml:\n {missing_in_file}") + + return loose_loot_distribution \ No newline at end of file diff --git a/src/static_loot.py b/src/static_loot.py new file mode 100644 index 0000000..e894013 --- /dev/null +++ b/src/static_loot.py @@ -0,0 +1,143 @@ +import json +import copy +import numpy as np +from itertools import groupby +from collections import defaultdict + + +def preprocess_staticloot(staticloot, static_weapon_ids): + containers = [] + for li in staticloot: + tpl = li["Items"][0]["_tpl"] + if tpl not in static_weapon_ids: + containers.append( + { + "type": tpl, + "containerId": li["Items"][0]["_id"], + "items": li["Items"][1:] + } + ) + return containers + + +class StaticLootProcessor: + def __init__(self, tarkov_items, static_weapon_ids, forced_static): + self._tarkov_items = tarkov_items + self._STATIC_WEAPON_IDS = static_weapon_ids + self._FORCED_STATIC = forced_static + + def create_static_containers(self, bio): + text = bio.read() + fi = json.loads(text) + + mapname = fi["data"]["Name"] + staticloot = [li for li in fi["data"]["Loot"] if li["IsStatic"]] + + s_containers = [] + s_weapons = [] + + staticloot = sorted(staticloot, key=lambda x: x["Id"]) + for li in staticloot: + tpl = li["Items"][0]["_tpl"] + if tpl in self._STATIC_WEAPON_IDS: + s_weapons.append(copy.deepcopy(li)) + else: + lic = copy.deepcopy(li) + lic["Root"] = None + lic["Items"] = [li["Items"][0]] + lic["Items"][0]["_id"] = None + s_containers.append(lic) + + if mapname in self._FORCED_STATIC: + s_forced = self._FORCED_STATIC[mapname] + else: + s_forced = [] + + static_containers = { + 'staticWeapons': s_weapons, + 'staticContainers': s_containers, + 'staticForced': s_forced + } + + return mapname, static_containers + + def create_ammo_distribution(self, container_counts): + ammo = [] + for ci in container_counts: + ammo += [ + item["_tpl"] for item in ci["items"] + if self._tarkov_items.is_baseclass(item["_tpl"], self._tarkov_items.BASECLASS.Ammo) + ] + ammo_types, ammo_counts = np.unique(ammo, return_counts=True) + caliber = [self._tarkov_items.ammo_caliber(ti) for ti in ammo_types] + ammo_counts = [ + { + "caliber": ci, + "tpl": ti, + "count": cnti + } + for ci, ti, cnti in zip(caliber, ammo_types, ammo_counts) + ] + ammo_counts = sorted(ammo_counts, key=lambda x: x["caliber"]) + caliber_group = {} + ammo_distribution = {} + for k, g in groupby(ammo_counts, lambda x: x["caliber"]): + caliber_group[k] = {} + g = list(g) + caliber_group[k]["tpl"] = [gi["tpl"] for gi in g] + caliber_group[k]["counts"] = [gi["count"] for gi in g] + + ammo_distribution[k] = [ + { + "tpl": gi["tpl"], + "relativeProbability": int(gi["count"]) + } + for gi in g + ] + return ammo_distribution + + def create_static_loot_distribution(self, container_counts): + static_loot_distribution = {} + types = np.unique([ci["type"] for ci in container_counts]) + idx = np.argsort(types) + types = types[idx] + for typei in types: + container_counts_selected = [ci for ci in container_counts if ci["type"] == typei] + + itemscounts = [] + for ci in container_counts_selected: + itemscounts.append(len([cii for cii in ci["items"] if cii["parentId"] == ci["containerId"]])) + + counts, relative_probability = np.unique(itemscounts, return_counts=True) + + static_loot_distribution[typei] = {} + static_loot_distribution[typei]["itemcountDistribution"] = [ + { + "count": int(cnt), + "relativeProbability": int(prb) + } + for cnt, prb in zip(counts, relative_probability) + ] + + itemscounts = defaultdict(int) + for ci in container_counts_selected: + for cii in ci["items"]: + if cii["parentId"] == ci["containerId"]: + itemscounts[cii["_tpl"]] += 1 + + itemids = [ti for ti, _ in itemscounts.items()] + itemcounts = [cnti for _, cnti in itemscounts.items()] + + idx = np.argsort(itemids) + itemids = np.array(itemids)[idx] + itemcounts = np.array(itemcounts)[idx] + + static_loot_distribution[typei]["itemDistribution"] = [ + { + "tpl": tpl, + "relativeProbability": int(prb) + } + for tpl, prb in zip(itemids, itemcounts) + ] + + return static_loot_distribution diff --git a/src/tarkov_items.py b/src/tarkov_items.py new file mode 100644 index 0000000..8625430 --- /dev/null +++ b/src/tarkov_items.py @@ -0,0 +1,188 @@ +# This is a sample Python script. +import json + + +class TarkovItems: + class BASECLASS: + DefaultInventory = "55d7217a4bdc2d86028b456d" + Inventory = "55d720f24bdc2d88028b456d" + Pockets = "557596e64bdc2dc2118b4571" + Weapon = "5422acb9af1c889c16000029" + Headwear = "5a341c4086f77401f2541505" + Armor = "5448e54d4bdc2dcc718b4568" + Vest = "5448e5284bdc2dcb718b4567" + Backpack = "5448e53e4bdc2d60728b4567" + Visors = "5448e5724bdc2ddf718b4568" + Food = "5448e8d04bdc2ddf718b4569" + Drink = "5448e8d64bdc2dce718b4568" + BarterItem = "5448eb774bdc2d0a728b4567" + Info = "5448ecbe4bdc2d60728b4568" + MedKit = "5448f39d4bdc2d0a728b4568" + Drugs = "5448f3a14bdc2d27728b4569" + Stimulator = "5448f3a64bdc2d60728b456a" + Medical = "5448f3ac4bdc2dce718b4569" + MedicalSupplies = "57864c8c245977548867e7f1" + Mod = "5448fe124bdc2da5018b4567" + FunctionalMod = "550aa4154bdc2dd8348b456b" + GearMod = "55802f3e4bdc2de7118b4584" + Stock = "55818a594bdc2db9688b456a" + Foregrip = "55818af64bdc2d5b648b4570" + MasterMod = "55802f4a4bdc2ddb688b4569" + Mount = "55818b224bdc2dde698b456f" + Muzzle = "5448fe394bdc2d0d028b456c" + Sights = "5448fe7a4bdc2d6f028b456b" + Meds = "543be5664bdc2dd4348b4569" + Money = "543be5dd4bdc2deb348b4569" + Key = "543be5e94bdc2df1348b4568" + KeyMechanical = "5c99f98d86f7745c314214b3" + Keycard = "5c164d2286f774194c5e69fa" + Equipment = "543be5f84bdc2dd4348b456a" + ThrowWeap = "543be6564bdc2df4348b4568" + FoodDrink = "543be6674bdc2df1348b4569" + Pistol = "5447b5cf4bdc2d65278b4567" + Smg = "5447b5e04bdc2d62278b4567" + AssaultRifle = "5447b5f14bdc2d61278b4567" + AssaultCarbine = "5447b5fc4bdc2d87278b4567" + Shotgun = "5447b6094bdc2dc3278b4567" + MarksmanRifle = "5447b6194bdc2d67278b4567" + SniperRifle = "5447b6254bdc2dc3278b4568" + MachineGun = "5447bed64bdc2d97278b4568" + GrenadeLauncher = "5447bedf4bdc2d87278b4568" + SpecialWeapon = "5447bee84bdc2dc3278b4569" + SpecItem = "5447e0e74bdc2d3c308b4567" + Knife = "5447e1d04bdc2dff2f8b4567" + Ammo = "5485a8684bdc2da71d8b4567" + AmmoBox = "543be5cb4bdc2deb348b4568" + LootContainer = "566965d44bdc2d814c8b4571" + MobContainer = "5448bf274bdc2dfc2f8b456a" + SearchableItem = "566168634bdc2d144c8b456c" + Stash = "566abbb64bdc2d144c8b457d" + SortingTable = "6050cac987d3f925bf016837" + LockableContainer = "5671435f4bdc2d96058b4569" + SimpleContainer = "5795f317245977243854e041" + StationaryContainer = "567583764bdc2d98058b456e" + Armband = "5b3f15d486f77432d0509248" + DogTagUsec = "59f32c3b86f77472a31742f0" + DogTagBear = "59f32bb586f774757e1e8442" + Jewelry = "57864a3d24597754843f8721" + Electronics = "57864a66245977548f04a81f" + BuildingMaterial = "57864ada245977548638de91" + Tool = "57864bb7245977548b3b66c2" + HouseholdGoods = "57864c322459775490116fbf" + Lubricant = "57864e4c24597754843f8723" + Battery = "57864ee62459775490116fc1" + FunctionalMod = "550aa4154bdc2dd8348b456b" + GearMod = "55802f3e4bdc2de7118b4584" + MasterMod = "55802f4a4bdc2ddb688b4569" + Other = "590c745b86f7743cc433c5f2" + AssaultScope = "55818add4bdc2d5b648b456f" + ReflexSight = "55818ad54bdc2ddc698b4569" + TacticalCombo = "55818b164bdc2ddc698b456c" + Magazine = "5448bc234bdc2d3c308b4569" + LightLaser = "55818b0e4bdc2dde698b456e" + Silencer = "550aa4cd4bdc2dd8348b456c" + PortableRangeFinder = "61605ddea09d851a0a0c1bbc" + Item = "54009119af1c881c07000029" + + def __init__(self, items: str, handbook: str, locales: str): + with open(items, encoding='utf-8') as fin: + self._items = json.load(fin) + with open(handbook, encoding='utf-8') as fin: + self._handbook = json.load(fin) + with open(locales, encoding='utf-8') as fin: + self._locales = json.load(fin) + + def is_baseclass(self, tpl, baseclass_id): + item_template = self._items[tpl] + if '_parent' not in item_template: + return False + if item_template['_parent'] == "": + return False + else: + parent_id = item_template['_parent'] + if parent_id == baseclass_id: + return True + else: + return self.is_baseclass(parent_id, baseclass_id) + + def is_questitem(self, tpl): + item_template = self._items[tpl] + return item_template["_props"]["QuestItem"] + + def max_durability(self, tpl): + item_template = self._items[tpl] + if "MaxDurability" in item_template["_props"]: + return item_template["_props"]["MaxDurability"] + else: + return None + + def ammo_caliber(self, tpl): + item_template = self._items[tpl] + if "Caliber" in item_template["_props"]: + return item_template["_props"]["Caliber"] + else: + return None + + def template(self, tpl): + return self._items[tpl] + + def template_locale_name(self, tpl): + id = self.template(tpl)['_id'] + return self._locales['templates'][id]['Name'] + + def ancestry(self, tpl): + ancestors = [] + item_template = self._items[tpl] + while '_parent' in item_template and item_template['_parent'] != "": + parent_id = item_template['_parent'] + parent = self._items[parent_id] + ancestors.append({'parentId': parent_id, "parentName": parent['_name']}) + item_template = parent + + return ancestors + + def price(self, tpl): + handbook_items = self._handbook["Items"] + return next((hi["Price"] for hi in handbook_items if hi["Id"] == tpl)) + + +if __name__ == "__main__": + tarkov_items = TarkovItems( + items="C:/GAMES/SPT_AKI_Dev/Server/project/assets/database/templates/items.json", + handbook="C:/GAMES/SPT_AKI_Dev/Server/project/assets/database/templates/handbook.json", + locales="C:/GAMES/SPT_AKI_Dev/Server/project/assets/database/locales/global/en.json" + ) + + print("a") + + # [ + # print(ti["_props"]["Caliber"]) + # for tpl, ti in tarkov_items._items.items() + # if tarkov_items.is_baseclass(tpl,tarkov_items.BASECLASS.Ammo) + # ] + + for tpl, ti in tarkov_items._items.items(): + if tarkov_items.is_baseclass(tpl, tarkov_items.BASECLASS.Magazine): + try: + ammoTpls = ti["_props"]["Cartridges"][0]["_props"]["filters"][0]["Filter"] + except Exception as e: + break + + ammos = [tarkov_items.template(tpli) for tpli in ammoTpls] + for ai in ammos: + try: + test = ai["_props"]["Caliber"] + print(test) + except Exception as e: + print(e) + print(ai) + + # print(tarkov_items.template_locale_name("59e6152586f77473dc057aa1")) + # + # print(tarkov_items.price("59e6152586f77473dc057aa1")) + # print(tarkov_items.ancestry("59e6152586f77473dc057aa1")) + # print(tarkov_items.is_baseclass("59e6152586f77473dc057aa1", TarkovItems.BASECLASS.Weapon)) + # + # print(tarkov_items.ancestry(TarkovItems.BASECLASS.Armor)) + # print(tarkov_items.ancestry(TarkovItems.BASECLASS.Headwear)) + # print(tarkov_items.ancestry(TarkovItems.BASECLASS.Vest))