initial commit

01_process_dump_data.py

@@ -0,0 +1,71 @@
+import sys
+import os
+import shutil
+import json
+import py7zr
+import yaml
+from tqdm import tqdm
+import subprocess
+
+from pathlib import Path
+
+with open('config/config.yaml', 'r') as fin:
+    config = yaml.load(fin, Loader=yaml.FullLoader)
+
+exe_7z = config["tools"]["7z_exe"]
+
+src = Path(config["archives"]["source_folder"])
+dst = Path(config["archives"]["target_folder"])
+
+archive_complete = Path(config["archives"]["backup_file"])
+
+used = {
+    dst / config["archives"]["loot_filename"]: lambda x: x.startswith("resp.client.location.getLocalloot"),
+    dst / config["archives"]["bot_filename"]: lambda x: x.startswith("resp.client.game.bot.generate"),
+}
+
+input_files = set(os.listdir(src))
+
+for archive_dst, filt in used.items():
+
+    mode = None
+    if os.path.exists(archive_dst):
+        mode = 'a'
+        with py7zr.SevenZipFile(archive_dst, 'r') as archive:
+            archive_files = set(archive.getnames())
+    else:
+        mode = 'w'
+        archive_files = []
+
+    # filter out every file which is already in the archive
+    files = input_files.difference(archive_files)
+
+    # get only those files which fit the recycle filters
+    print(f"Number of files which are not found in the archive: {len(files)}")
+    files = [
+        fi for fi in files
+        if filt(fi)
+    ]
+    print(f"Of those according to filter add {len(files)} files to {archive_dst}")
+
+    # temporary copy to destination folder to add them via 7z subprocess
+    if len(files) > 0:
+        for file in tqdm(files):
+            shutil.copy(src / file, dst / file)
+
+        cmd = f"{exe_7z} a -t7z {archive_dst} *.json"
+        os.chdir(dst)
+        subprocess.call(cmd)
+
+        for file in tqdm(files):
+            os.remove(dst / file)
+
+# add all files to backup archive (if they are not inside already)
+cmd = f"{exe_7z} a -up1q1r2x1y1z1w1 -t7z {archive_complete} *.json"
+os.chdir(src)
+subprocess.call(cmd)
+
+# delete raw data from source folder
+files = os.listdir(src)
+for file in tqdm(files):
+    os.remove(src / file)

02_loot_generator_parallel.py

@@ -0,0 +1,219 @@
+import json
+import yaml
+import time
+import py7zr
+import hashlib
+import datetime
+
+from tqdm import tqdm
+from pathlib import Path
+
+from concurrent import futures
+
+from collections import defaultdict
+from itertools import groupby
+
+from src.tarkov_items import TarkovItems
+
+from src.static_loot import preprocess_staticloot, StaticLootProcessor
+from src.loose_loot import preprocess_looseloot, LooseLootProcessor
+
+with open('config/config.yaml', 'r') as fin:
+    config = yaml.load(fin, Loader=yaml.FullLoader)
+
+with open(f'config/{config["config"]["static"]["forced_static_yaml"]}', 'r') as fin:
+    FORCED_STATIC = yaml.load(fin, Loader=yaml.FullLoader)
+
+with open(f'config/{config["config"]["static"]["forced_loose_yaml"]}', 'r') as fin:
+    FORCED_LOOSE = yaml.load(fin, Loader=yaml.FullLoader)
+
+with open(f'config/{config["server"]["map_directory_mapping_yaml"]}', 'r') as fin:
+    loose_loot_dir_map = yaml.load(fin, Loader=yaml.FullLoader)
+
+STATIC_WEAPON_IDS = config["config"]["static"]["static_weapon_ids"]
+
+tarkov_server_dir = Path(config["server"]["location"])
+loot_dump_archive = Path(config["archives"]["target_folder"]) / config["archives"]["loot_filename"]
+
+
+def hash_file(text):
+    sha256 = hashlib.sha256()
+    sha256.update(text)
+    return sha256.hexdigest()
+
+
+def parse_dumps(input):
+    fname = input[0]
+    bio = input[1]
+    text = bio.read()
+
+    fi = json.loads(text)
+
+    datestr = fname.split(".getLocalloot_")[-1].split(".")[0]
+    date = datetime.datetime.strptime(datestr, "%Y-%m-%d_%H-%M-%S")
+
+    if fi["data"] is not None:
+        basic_info = {
+            "map": fi["data"]["Name"],
+            "filehash": hash_file(text),
+            "date": date,
+            "fname": fname
+        }
+
+        looseloot = [li for li in fi["data"]["Loot"] if not li["IsStatic"]]
+        staticloot = [li for li in fi["data"]["Loot"] if li["IsStatic"]]
+
+        looseloot_processed = preprocess_looseloot(looseloot)
+        containers = preprocess_staticloot(staticloot, STATIC_WEAPON_IDS)
+
+        return {
+            "basic_info": basic_info,
+            "looseloot": looseloot_processed,
+            "containers": containers
+        }
+    else:
+        return None
+
+
+def main():
+    tarkov_items = TarkovItems(
+        items=tarkov_server_dir / "project/assets/database/templates/items.json",
+        handbook=tarkov_server_dir / "project/assets/database/templates/handbook.json",
+        locales=tarkov_server_dir / "project/assets/database/locales/global/en.json"
+    )
+
+    loose_loot_dir = tarkov_server_dir / "project/assets/database/locations"
+    static_loot_dir = tarkov_server_dir / "project/assets/database/loot"
+
+    print("Open dump archive", end="; ")
+    map_files = {}
+    gather_loot_results = []
+    time_start = time.time()
+    with py7zr.SevenZipFile(loot_dump_archive, 'r') as archive:
+        archive_files = set(archive.getnames())
+        with futures.ProcessPoolExecutor() as executor:
+            print("Gathering dumps")
+            for result in list(tqdm(executor.map(parse_dumps, archive.read(archive_files).items()), total=len(archive_files))):
+                if result is not None:
+                    gather_loot_results.append(result)
+                    # get the newest dump per map
+                    mapi = result["basic_info"]["map"]
+                    if mapi not in map_files:
+                        map_files[mapi] = (
+                            result["basic_info"]["date"],
+                            result["basic_info"]["fname"]
+                        )
+                    else:
+                        if result["basic_info"]["date"] > map_files[mapi][0]:
+                            map_files[mapi] = (
+                                result["basic_info"]["date"],
+                                result["basic_info"]["fname"]
+                            )
+
+
+    print(f"Reading dumps took {time.time() - time_start} seconds.")
+    dump_count = len(gather_loot_results)
+
+    # remove duplicate dumps
+    time_start = time.time()
+    gather_loot_results = sorted(gather_loot_results, key=lambda x: x["basic_info"]["filehash"])
+    gather_loot_results_unique = []
+    for _, g in groupby(gather_loot_results, key=lambda x: x["basic_info"]["filehash"]):
+        g = list(g)
+        if len(g) > 1:
+            # print(f"Duplicate dumps: {', '.join([gi['filename'] for gi in g])}")
+            pass
+        gather_loot_results_unique.append(g[0])
+
+    del gather_loot_results
+    dump_count_unique = len(gather_loot_results_unique)
+
+    print(
+        f"Removing duplicates took {time.time() - time_start} seconds: {dump_count - dump_count_unique} / {dump_count}")
+
+    # Map Reduce
+    print("Map reducing dumps", end="; ")
+    time_start = time.time()
+    gather_loot_results_unique = sorted(gather_loot_results_unique, key=lambda x: x["basic_info"]["map"])
+    looseloot_counts = {}
+    container_counts = []
+    map_counts = {}
+    for mapi, g in groupby(gather_loot_results_unique, key=lambda x: x["basic_info"]["map"]):
+        g = list(g)
+        map_counts[mapi] = len(g)
+        looseloot_counts[mapi] = {}
+
+        looseloot_counts[mapi]["counts"] = defaultdict(int)
+        looseloot_counts[mapi]["items"] = defaultdict(list)
+        looseloot_counts[mapi]["itemproperties"] = defaultdict(list)
+        looseloot_counts[mapi]["map_spawnpoint_count"] = []
+
+        for gi in g:
+
+            container_counts += gi["containers"]
+
+            for k, v in gi["looseloot"]["counts"].items():
+                looseloot_counts[mapi]["counts"][k] += v
+            for k, v in gi["looseloot"]["items"].items():
+                looseloot_counts[mapi]["items"][k] += v
+            for k, v in gi["looseloot"]["itemproperties"].items():
+                looseloot_counts[mapi]["itemproperties"][k] += v
+
+            looseloot_counts[mapi]["map_spawnpoint_count"] += [gi["looseloot"]["map_spawnpoint_count"]]
+
+    del gather_loot_results_unique
+    print(f"took {time.time() - time_start} seconds.")
+
+    static_loot_processor = StaticLootProcessor(
+        tarkov_items=tarkov_items,
+        static_weapon_ids=STATIC_WEAPON_IDS,
+        forced_static=FORCED_STATIC
+    )
+    # create static containers (containers per map, forced loot in map, static weapons in map)
+    print("Create \"static containers\"", end='; ')
+    time_start = time.time()
+    static_containers = {}
+    with py7zr.SevenZipFile(loot_dump_archive, 'r') as archive:
+        targets = [datename_tuple[1] for _, datename_tuple in map_files.items()]
+        targets = sorted(targets)
+        for fname, bio in archive.read(targets).items():
+            mapi, static_containers_mi = static_loot_processor.create_static_containers(bio)
+            static_containers[mapi] = static_containers_mi
+    print(f"took {time.time() - time_start} seconds.")
+    with open(static_loot_dir / "staticContainers.json", "w") as fout:
+        json.dump(static_containers, fout, indent=1)
+
+    # Ammo distribution
+    time_start = time.time()
+    print(f"Creating \"ammo\" distribution", end="; ")
+    ammo_distribution = static_loot_processor.create_ammo_distribution(container_counts)
+    print(f"took {time.time() - time_start} seconds.")
+    with open(static_loot_dir / "staticAmmo.json", "w") as fout:
+        json.dump(ammo_distribution, fout, indent=1)
+
+    # Static loot distribution
+    time_start = time.time()
+    print(f"Creating \"static container\"", end='; ')
+    static_loot_distribution = static_loot_processor.create_static_loot_distribution(container_counts)
+    print(f"took {time.time() - time_start} seconds.")
+    with open(static_loot_dir / "staticLoot.json", 'w') as fout:
+        json.dump(static_loot_distribution, fout, indent=1)
+
+    # Loose loot distribution
+    loose_loot_processor = LooseLootProcessor(
+        tarkov_items=tarkov_items,
+        FORCED_LOOSE=FORCED_LOOSE
+    )
+    time_start = time.time()
+    print(f"Calculating \"loose loot\" distribution", end='; ')
+    loose_loot_distribution = loose_loot_processor.create_loose_loot_distribution(map_counts, looseloot_counts)
+    print(f"took {time.time() - time_start} seconds")
+
+    for mi, cnt in map_counts.items():
+        for mapdir in loose_loot_dir_map[mi]:
+            with open(loose_loot_dir / mapdir / "looseLoot.json", "w") as fout:
+                json.dump(loose_loot_distribution[mi], fout, indent=1)
+
+
+if __name__ == '__main__':
+    main()

config/config.yaml

@@ -0,0 +1,30 @@
+---
+tools:
+  7z_exe: 'C:/Program Files/7-Zip/7z.exe'
+server:
+  # SPT-AKI Server location (to store the generated json)
+  location: 'C:/GAMES/SPT_AKI_Dev/Server/'
+  # The map names in the loot dump do not directly corresond to the server database structure, this is the mapping
+  map_directory_mapping_yaml:  map_directory_mapping.yaml
+
+archives:
+  # where we put our sequential dump jsons to be sorted archived, preprocessor will put them into archives
+  source_folder: 'D:/Games/SPT_Dev/map_dumps/12.12/'
+  # here we store the archives with used data
+  target_folder: 'D:/Games/SPT_Dev/map_dumps/12.12_used/'
+  # archive name for map loot dump data
+  loot_filename: loot_dumps.7z
+  # archive name for bot dump data
+  bot_filename: bot_dumps.7z
+  # all dump data is appended into an archive in this folder
+  backup_file: 'D:/Games/SPT_Dev/map_dumps/12.12_unused/complete.7z'
+config:
+  static:
+    # ids for static weapons
+    static_weapon_ids:
+      - 5d52cc5ba4b9367408500062
+      - 5cdeb229d7f00c000e7ce174
+    # we have a separate yaml that defined forced static loot per map
+    forced_static_yaml: forced_static.yaml
+    # known quest items per map for validation
+    forced_loose_yaml: forced_loose.yaml

config/forced_loose.yaml

@@ -0,0 +1,72 @@
+---
+Customs:
+- itemTpl: 5938188786f77474f723e87f  # Case 0031
+- itemTpl: 5c12301c86f77419522ba7e4  # Flash drive with fake info
+- itemTpl: 593965cf86f774087a77e1b6  # Case 0048
+- itemTpl: 591092ef86f7747bb8703422  # Secure folder 0022 in big red offices
+- itemTpl: 590c62a386f77412b0130255  # Sliderkey Secure Flash drive in Dorms 2-way room 220
+- itemTpl: 5939e9b286f77462a709572c  # Sealed letter (Terragroup)
+- itemTpl: 5ac620eb86f7743a8e6e0da0  # Package of graphics cards in big red offices
+- itemTpl: 590dde5786f77405e71908b2  # Bank case
+- itemTpl: 5910922b86f7747d96753483  # Carbon case
+- itemTpl: 5937fd0086f7742bf33fc198  # Bronze pocket watch on a chain
+- itemTpl: 5939a00786f7742fe8132936  # Golden Zibbo lighter
+- itemTpl: 5939e5a786f77461f11c0098  # Secure Folder 0013
+
+Woods:
+- itemTpl: 5938878586f7741b797c562f  # Case 0052
+- itemTpl: 5d3ec50586f774183a607442  # Jaeger's message Underneath the wooden lookout post.
+- itemTpl: 5af04e0a86f7743a532b79e2  # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope
+- itemTpl: 5a687e7886f7740c4a5133fb  # Blood sample
+- itemTpl: 5af04c0b86f774138708f78e  # Motor Controller: item_barter_electr_controller
+
+Shoreline:
+- itemTpl: 5a294d7c86f7740651337cf9  # Drone 1 SAS disk
+- itemTpl: 5a294d8486f774068638cd93  # Drone 2 SAS disk: ambiguous with itemTpl 5a294f1686f774340c7b7e4a
+- itemTpl: 5efdafc1e70b5e33f86de058  # Sanitar's Surgery kit marked with a blue symbol
+- itemTpl: 5939e5a786f77461f11c0098  # Secure folder 0013
+- itemTpl: 5a6860d886f77411cd3a9e47  # Secure folder 0060
+- itemTpl: 5a29357286f77409c705e025  # Sliderkey Flash drive
+- itemTpl: 5efdaf6de6a30218ed211a48  # Sanitar's Ophthalmoscope In potted plant on dining room table.
+- itemTpl: 5d357d6b86f7745b606e3508  # Photo album in west wing room 303
+- itemTpl: 5b4c72b386f7745b453af9c0  # Motor Controller: item_barter_electr_controller2
+- itemTpl: 5a0448bc86f774736f14efa8  # Key to the closed premises of the Health Resort
+- itemTpl: 5a29276886f77435ed1b117c  # Working hard drive
+- itemTpl: 5b4c72fb86f7745cef1cffc5  # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope2
+- itemTpl: 5b4c72c686f77462ac37e907  # Motor Controller: item_barter_electr_controller3
+- itemTpl: 5b43237186f7742f3a4ab252  # Chemical container: item_quest_chem_container
+- itemTpl: 5a29284f86f77463ef3db363  # Toughbook reinforced laptop
+
+
+Interchange:
+- itemTpl: 5ae9a18586f7746e381e16a3  # OLI cargo manifests
+- itemTpl: 5ae9a0dd86f7742e5f454a05  # Goshan cargo manifests
+- itemTpl: 5ae9a1b886f77404c8537c62  # Idea cargo manifests
+- itemTpl: 5ae9a25386f7746dd946e6d9  # OLI cargo route documents (locked)
+- itemTpl: 5ae9a3f586f7740aab00e4e6  # Clothes design handbook - Part 1
+- itemTpl: 5ae9a4fc86f7746e381e1753  # Clothes design handbook - Part 2
+- itemTpl: 5b4c81a086f77417d26be63f  # Chemical container item_quest_chem_container2
+- itemTpl: 5b4c81bd86f77418a75ae159  # Chemical container item_quest_chem_container3
+
+Factory:
+- itemTpl: 591093bb86f7747caa7bb2ee  # On the neck of the dead scav in the bunker
+- itemTpl: 593a87af86f774122f54a951  # Syringe with a chemical
+
+Lighthouse:
+- itemTpl: 61904c9df62c89219a56e034  # The message is tucked under the bottom of the door to the cabin.
+- itemTpl: 619268ad78f4fa33f173dbe5  # Water pump operation data On the desk between other documents in the upper office.
+- itemTpl: 619268de2be33f2604340159  # Pumping Station Operation Data In the upper floor office on the shelf.
+- itemTpl: 61a00bcb177fb945751bbe6a  # Stolen military documents On the back corner of the dining room table on the third floor at Chalet.
+- itemTpl: 619252352be33f26043400a7  # Laptop with information
+
+ReserveBase:
+- itemTpl: 60915994c49cf53e4772cc38  # Military documents 1 on the table inside bunker control room
+- itemTpl: 60a3b6359c427533db36cf84  # Military documents 2 On the bottom shelf of the cupboard near the corner.
+- itemTpl: 60a3b65c27adf161da7b6e14  # Military documents 3 Inside the cupboard next to the 4x4 Weapon Box.
+- itemTpl: 608c22a003292f4ba43f8a1a  # Medical record 1 (locked by RB-KSM key)
+- itemTpl: 60a3b5b05f84d429b732e934  # Medical record 2 (locked by RB-SMP key)
+- itemTpl: 609267a2bb3f46069c3e6c7d  # T-90M Commander Control Panel
+- itemTpl: 60c080eb991ac167ad1c3ad4  # MBT Integrated Navigation System
+
+Laboratory:
+- itemTpl: 5eff135be0d3331e9d282b7b  # Flash drive marked with blue tape

config/forced_static.yaml

@@ -0,0 +1,5 @@
+---
+Customs:
+# unknown key
+- containerId: custom_multiScene_00058
+  itemTpl: 593962ca86f774068014d9af

config/map_directory_mapping.yaml

@@ -0,0 +1,18 @@
+---
+Customs:
+- bigmap
+Factory:
+- factory4_day
+- factory4_night
+Interchange:
+- interchange
+Laboratory:
+- laboratory
+Lighthouse:
+- lighthouse
+ReserveBase:
+- rezervbase
+Shoreline:
+- shoreline
+Woods:
+- woods

src/loose_loot.py

@@ -0,0 +1,157 @@
+import json
+import copy
+import numpy as np
+from itertools import groupby
+from collections import defaultdict
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from src.tarkov_items import TarkovItems
+
+def preprocess_looseloot(looseloot):
+    looseloot_ci = {}
+    looseloot_ci["counts"] = defaultdict(int)
+    looseloot_ci["items"] = defaultdict(list)
+    looseloot_ci["itemproperties"] = defaultdict(list)
+    looseloot_ci["map_spawnpoint_count"] = len(looseloot)
+
+    unique_ids = {}
+    for li in looseloot:
+
+        group_fun = lambda x: (
+            x["Position"]["x"],
+            x["Position"]["y"],
+            x["Position"]["z"],
+            x["Rotation"]["x"],
+            x["Rotation"]["y"],
+            x["Rotation"]["z"],
+            x["useGravity"],
+            x["IsGroupPosition"],
+        )
+
+        # the bsg ids are insane.
+        # Sometimes the last 7 digits vary but they spawn the same item at the same position
+        # e.g. for the quest item "60a3b65c27adf161da7b6e14" at "loot_bunker_quest (3)555192"
+        # so the first approach was to remove the last digits.
+        # We then saw, that sometimes when the last digits differ for the same string, also the position
+        # differs.
+        # We decided to group over the position/rotation/useGravity since they make out a distinct spot
+        sane_id = str(group_fun(li))
+        if sane_id not in unique_ids:
+            unique_ids[sane_id] = li["Id"]
+            looseloot_ci["counts"][sane_id] += 1
+        else:
+            pass
+            # print(f'Spawn Points dupe: {fname} {unique_ids[sane_id]} = {li["Id"]}')
+            # continue
+
+        looseloot_ci["items"][sane_id].append(li["Items"][0]["_tpl"])
+        looseloot_ci["itemproperties"][sane_id].append(li)
+
+    return looseloot_ci
+
+
+class LooseLootProcessor:
+
+    def __init__(self, tarkov_items: 'TarkovItems', FORCED_LOOSE):
+        self._tarkov_items = tarkov_items
+        self._FORCED_LOOSE = FORCED_LOOSE
+
+    def create_loose_loot_distribution(self, map_counts, looseloot_counts):
+        probabilities = {}
+        loose_loot_distribution = {}
+        for mi, map_cnt in map_counts.items():
+            probabilities[mi] = {}
+            for idi, cnt in looseloot_counts[mi]["counts"].items():
+                probabilities[mi][idi] = cnt / map_cnt
+
+            loose_loot_distribution[mi] = {
+                "spawnpointCount": {
+                    "mean": np.mean(looseloot_counts[mi]["map_spawnpoint_count"]),
+                    "std": np.std(looseloot_counts[mi]["map_spawnpoint_count"])
+                },
+                "spawnpointsForced": [],
+                "spawnpoints": []
+            }
+
+            for spawnpoint, itemlist in looseloot_counts[mi]["itemproperties"].items():
+                itemscounts = defaultdict(int)
+
+                for item in itemlist:
+                    itemscounts[item["Items"][0]["_tpl"]] += 1
+
+                # Group by arguments to create possible positions / rotations per spawnpoint
+                group_fun = lambda x: (
+                    x["Position"]["x"],
+                    x["Position"]["y"],
+                    x["Position"]["z"],
+                    x["Rotation"]["x"],
+                    x["Rotation"]["y"],
+                    x["Rotation"]["z"],
+                    x["useGravity"],
+                    x["GroupPositions"],
+                    x["IsGroupPosition"],
+                )
+
+                # check if grouping is unique
+                itemlist_sorted = sorted(itemlist, key=group_fun)
+                itemlist_grouped = groupby(itemlist_sorted, group_fun)
+                ks = []
+                for k, g in itemlist_grouped:
+                    ks.append(k)
+                    gl = list(g)
+                if len(ks) > 1:
+                    print(ks)
+                    print(json.dumps(gl, indent=4))
+                    raise ValueError("Attributes for distinct SpawnPointId differ")
+
+                template = copy.deepcopy(gl[0])
+                template["Root"] = None
+
+                item_distribution = [
+                    {
+                        "tpl": tpl,
+                        "relativeProbability": cnt,
+                    }
+                    for tpl, cnt in itemscounts.items()
+                ]
+
+                if any((self._tarkov_items.is_questitem(item['tpl']) for item in item_distribution)):
+                    spawnpoint = {
+                        "probability": probabilities[mi][spawnpoint],
+                        "template": template
+                    }
+                    loose_loot_distribution[mi]["spawnpointsForced"].append(spawnpoint)
+                elif probabilities[mi][spawnpoint] > 0.99:
+                    spawnpoint = {
+                        "probability": probabilities[mi][spawnpoint],
+                        "template": template
+                    }
+                    loose_loot_distribution[mi]["spawnpointsForced"].append(spawnpoint)
+                    print(f"Warning: High probability if spawnpoint {template['Id']} even though no quest item found inside it")
+                else:
+                    template["Items"] = []
+
+                    spawnpoint = {
+                        "probability": probabilities[mi][spawnpoint],
+                        "template": template,
+                        "itemDistribution": sorted(item_distribution, key=lambda x: x["tpl"])
+                    }
+                    loose_loot_distribution[mi]["spawnpoints"].append(spawnpoint)
+
+            loose_loot_distribution[mi]["spawnpoints"] = sorted(loose_loot_distribution[mi]["spawnpoints"], key=lambda x: x["template"]["Id"])
+
+            # Cross check with forced loot in dumps vs items defined in forced_loose.yaml
+            forced_tpls_file = set([forceditem["itemTpl"] for forceditem in self._FORCED_LOOSE[mi]])
+            forced_tpls_found = set([forceditem["template"]["Items"][0]["_tpl"] for forceditem in loose_loot_distribution[mi]["spawnpointsForced"]])
+
+            # all the tpls that are defined in the forced_loose.yaml for this map that are not found as forced
+            missing_in_dumps = forced_tpls_file - forced_tpls_found
+            if len(missing_in_dumps) > 0:
+                print(f"Error: {mi} Items defined in forced_loose.yaml were not found as quest item:\n {missing_in_dumps}")
+            # all the tpls that are found as forced but not in the forced_loose.yaml
+            missing_in_file = forced_tpls_found - forced_tpls_file
+            if len(missing_in_file) > 0:
+                print(f"Warning: {mi} Items were found as forced in dump but were not defined in forced_loose.yaml:\n {missing_in_file}")
+
+        return loose_loot_distribution

src/static_loot.py

@@ -0,0 +1,143 @@
+import json
+import copy
+import numpy as np
+from itertools import groupby
+from collections import defaultdict
+
+
+def preprocess_staticloot(staticloot, static_weapon_ids):
+    containers = []
+    for li in staticloot:
+        tpl = li["Items"][0]["_tpl"]
+        if tpl not in static_weapon_ids:
+            containers.append(
+                {
+                    "type": tpl,
+                    "containerId": li["Items"][0]["_id"],
+                    "items": li["Items"][1:]
+                }
+            )
+    return containers
+
+
+class StaticLootProcessor:
+    def __init__(self, tarkov_items, static_weapon_ids, forced_static):
+        self._tarkov_items = tarkov_items
+        self._STATIC_WEAPON_IDS = static_weapon_ids
+        self._FORCED_STATIC = forced_static
+
+    def create_static_containers(self, bio):
+        text = bio.read()
+        fi = json.loads(text)
+
+        mapname = fi["data"]["Name"]
+        staticloot = [li for li in fi["data"]["Loot"] if li["IsStatic"]]
+
+        s_containers = []
+        s_weapons = []
+
+        staticloot = sorted(staticloot, key=lambda x: x["Id"])
+        for li in staticloot:
+            tpl = li["Items"][0]["_tpl"]
+            if tpl in self._STATIC_WEAPON_IDS:
+                s_weapons.append(copy.deepcopy(li))
+            else:
+                lic = copy.deepcopy(li)
+                lic["Root"] = None
+                lic["Items"] = [li["Items"][0]]
+                lic["Items"][0]["_id"] = None
+                s_containers.append(lic)
+
+        if mapname in self._FORCED_STATIC:
+            s_forced = self._FORCED_STATIC[mapname]
+        else:
+            s_forced = []
+
+        static_containers = {
+            'staticWeapons': s_weapons,
+            'staticContainers': s_containers,
+            'staticForced': s_forced
+        }
+
+        return mapname, static_containers
+
+    def create_ammo_distribution(self, container_counts):
+        ammo = []
+        for ci in container_counts:
+            ammo += [
+                item["_tpl"] for item in ci["items"]
+                if self._tarkov_items.is_baseclass(item["_tpl"], self._tarkov_items.BASECLASS.Ammo)
+            ]
+        ammo_types, ammo_counts = np.unique(ammo, return_counts=True)
+        caliber = [self._tarkov_items.ammo_caliber(ti) for ti in ammo_types]
+        ammo_counts = [
+            {
+                "caliber": ci,
+                "tpl": ti,
+                "count": cnti
+            }
+            for ci, ti, cnti in zip(caliber, ammo_types, ammo_counts)
+        ]
+        ammo_counts = sorted(ammo_counts, key=lambda x: x["caliber"])
+        caliber_group = {}
+        ammo_distribution = {}
+        for k, g in groupby(ammo_counts, lambda x: x["caliber"]):
+            caliber_group[k] = {}
+            g = list(g)
+            caliber_group[k]["tpl"] = [gi["tpl"] for gi in g]
+            caliber_group[k]["counts"] = [gi["count"] for gi in g]
+
+            ammo_distribution[k] = [
+                {
+                    "tpl": gi["tpl"],
+                    "relativeProbability": int(gi["count"])
+                }
+                for gi in g
+            ]
+        return ammo_distribution
+
+    def create_static_loot_distribution(self, container_counts):
+        static_loot_distribution = {}
+        types = np.unique([ci["type"] for ci in container_counts])
+        idx = np.argsort(types)
+        types = types[idx]
+        for typei in types:
+            container_counts_selected = [ci for ci in container_counts if ci["type"] == typei]
+
+            itemscounts = []
+            for ci in container_counts_selected:
+                itemscounts.append(len([cii for cii in ci["items"] if cii["parentId"] == ci["containerId"]]))
+
+            counts, relative_probability = np.unique(itemscounts, return_counts=True)
+
+            static_loot_distribution[typei] = {}
+            static_loot_distribution[typei]["itemcountDistribution"] = [
+                {
+                    "count": int(cnt),
+                    "relativeProbability": int(prb)
+                }
+                for cnt, prb in zip(counts, relative_probability)
+            ]
+
+            itemscounts = defaultdict(int)
+            for ci in container_counts_selected:
+                for cii in ci["items"]:
+                    if cii["parentId"] == ci["containerId"]:
+                        itemscounts[cii["_tpl"]] += 1
+
+            itemids = [ti for ti, _ in itemscounts.items()]
+            itemcounts = [cnti for _, cnti in itemscounts.items()]
+
+            idx = np.argsort(itemids)
+            itemids = np.array(itemids)[idx]
+            itemcounts = np.array(itemcounts)[idx]
+
+            static_loot_distribution[typei]["itemDistribution"] = [
+                {
+                    "tpl": tpl,
+                    "relativeProbability": int(prb)
+                }
+                for tpl, prb in zip(itemids, itemcounts)
+            ]
+
+        return static_loot_distribution

src/tarkov_items.py

@@ -0,0 +1,188 @@
+# This is a sample Python script.
+import json
+
+
+class TarkovItems:
+    class BASECLASS:
+        DefaultInventory = "55d7217a4bdc2d86028b456d"
+        Inventory = "55d720f24bdc2d88028b456d"
+        Pockets = "557596e64bdc2dc2118b4571"
+        Weapon = "5422acb9af1c889c16000029"
+        Headwear = "5a341c4086f77401f2541505"
+        Armor = "5448e54d4bdc2dcc718b4568"
+        Vest = "5448e5284bdc2dcb718b4567"
+        Backpack = "5448e53e4bdc2d60728b4567"
+        Visors = "5448e5724bdc2ddf718b4568"
+        Food = "5448e8d04bdc2ddf718b4569"
+        Drink = "5448e8d64bdc2dce718b4568"
+        BarterItem = "5448eb774bdc2d0a728b4567"
+        Info = "5448ecbe4bdc2d60728b4568"
+        MedKit = "5448f39d4bdc2d0a728b4568"
+        Drugs = "5448f3a14bdc2d27728b4569"
+        Stimulator = "5448f3a64bdc2d60728b456a"
+        Medical = "5448f3ac4bdc2dce718b4569"
+        MedicalSupplies = "57864c8c245977548867e7f1"
+        Mod = "5448fe124bdc2da5018b4567"
+        FunctionalMod = "550aa4154bdc2dd8348b456b"
+        GearMod = "55802f3e4bdc2de7118b4584"
+        Stock = "55818a594bdc2db9688b456a"
+        Foregrip = "55818af64bdc2d5b648b4570"
+        MasterMod = "55802f4a4bdc2ddb688b4569"
+        Mount = "55818b224bdc2dde698b456f"
+        Muzzle = "5448fe394bdc2d0d028b456c"
+        Sights = "5448fe7a4bdc2d6f028b456b"
+        Meds = "543be5664bdc2dd4348b4569"
+        Money = "543be5dd4bdc2deb348b4569"
+        Key = "543be5e94bdc2df1348b4568"
+        KeyMechanical = "5c99f98d86f7745c314214b3"
+        Keycard = "5c164d2286f774194c5e69fa"
+        Equipment = "543be5f84bdc2dd4348b456a"
+        ThrowWeap = "543be6564bdc2df4348b4568"
+        FoodDrink = "543be6674bdc2df1348b4569"
+        Pistol = "5447b5cf4bdc2d65278b4567"
+        Smg = "5447b5e04bdc2d62278b4567"
+        AssaultRifle = "5447b5f14bdc2d61278b4567"
+        AssaultCarbine = "5447b5fc4bdc2d87278b4567"
+        Shotgun = "5447b6094bdc2dc3278b4567"
+        MarksmanRifle = "5447b6194bdc2d67278b4567"
+        SniperRifle = "5447b6254bdc2dc3278b4568"
+        MachineGun = "5447bed64bdc2d97278b4568"
+        GrenadeLauncher = "5447bedf4bdc2d87278b4568"
+        SpecialWeapon = "5447bee84bdc2dc3278b4569"
+        SpecItem = "5447e0e74bdc2d3c308b4567"
+        Knife = "5447e1d04bdc2dff2f8b4567"
+        Ammo = "5485a8684bdc2da71d8b4567"
+        AmmoBox = "543be5cb4bdc2deb348b4568"
+        LootContainer = "566965d44bdc2d814c8b4571"
+        MobContainer = "5448bf274bdc2dfc2f8b456a"
+        SearchableItem = "566168634bdc2d144c8b456c"
+        Stash = "566abbb64bdc2d144c8b457d"
+        SortingTable = "6050cac987d3f925bf016837"
+        LockableContainer = "5671435f4bdc2d96058b4569"
+        SimpleContainer = "5795f317245977243854e041"
+        StationaryContainer = "567583764bdc2d98058b456e"
+        Armband = "5b3f15d486f77432d0509248"
+        DogTagUsec = "59f32c3b86f77472a31742f0"
+        DogTagBear = "59f32bb586f774757e1e8442"
+        Jewelry = "57864a3d24597754843f8721"
+        Electronics = "57864a66245977548f04a81f"
+        BuildingMaterial = "57864ada245977548638de91"
+        Tool = "57864bb7245977548b3b66c2"
+        HouseholdGoods = "57864c322459775490116fbf"
+        Lubricant = "57864e4c24597754843f8723"
+        Battery = "57864ee62459775490116fc1"
+        FunctionalMod = "550aa4154bdc2dd8348b456b"
+        GearMod = "55802f3e4bdc2de7118b4584"
+        MasterMod = "55802f4a4bdc2ddb688b4569"
+        Other = "590c745b86f7743cc433c5f2"
+        AssaultScope = "55818add4bdc2d5b648b456f"
+        ReflexSight = "55818ad54bdc2ddc698b4569"
+        TacticalCombo = "55818b164bdc2ddc698b456c"
+        Magazine = "5448bc234bdc2d3c308b4569"
+        LightLaser = "55818b0e4bdc2dde698b456e"
+        Silencer = "550aa4cd4bdc2dd8348b456c"
+        PortableRangeFinder = "61605ddea09d851a0a0c1bbc"
+        Item = "54009119af1c881c07000029"
+
+    def __init__(self, items: str, handbook: str, locales: str):
+        with open(items, encoding='utf-8') as fin:
+            self._items = json.load(fin)
+        with open(handbook, encoding='utf-8') as fin:
+            self._handbook = json.load(fin)
+        with open(locales, encoding='utf-8') as fin:
+            self._locales = json.load(fin)
+
+    def is_baseclass(self, tpl, baseclass_id):
+        item_template = self._items[tpl]
+        if '_parent' not in item_template:
+            return False
+        if item_template['_parent'] == "":
+            return False
+        else:
+            parent_id = item_template['_parent']
+            if parent_id == baseclass_id:
+                return True
+            else:
+                return self.is_baseclass(parent_id, baseclass_id)
+
+    def is_questitem(self, tpl):
+        item_template = self._items[tpl]
+        return item_template["_props"]["QuestItem"]
+
+    def max_durability(self, tpl):
+        item_template = self._items[tpl]
+        if "MaxDurability" in item_template["_props"]:
+            return item_template["_props"]["MaxDurability"]
+        else:
+            return None
+
+    def ammo_caliber(self, tpl):
+        item_template = self._items[tpl]
+        if "Caliber" in item_template["_props"]:
+            return item_template["_props"]["Caliber"]
+        else:
+            return None
+
+    def template(self, tpl):
+        return self._items[tpl]
+
+    def template_locale_name(self, tpl):
+        id = self.template(tpl)['_id']
+        return self._locales['templates'][id]['Name']
+
+    def ancestry(self, tpl):
+        ancestors = []
+        item_template = self._items[tpl]
+        while '_parent' in item_template and item_template['_parent'] != "":
+            parent_id = item_template['_parent']
+            parent = self._items[parent_id]
+            ancestors.append({'parentId': parent_id, "parentName": parent['_name']})
+            item_template = parent
+
+        return ancestors
+
+    def price(self, tpl):
+        handbook_items = self._handbook["Items"]
+        return next((hi["Price"] for hi in handbook_items if hi["Id"] == tpl))
+
+
+if __name__ == "__main__":
+    tarkov_items = TarkovItems(
+        items="C:/GAMES/SPT_AKI_Dev/Server/project/assets/database/templates/items.json",
+        handbook="C:/GAMES/SPT_AKI_Dev/Server/project/assets/database/templates/handbook.json",
+        locales="C:/GAMES/SPT_AKI_Dev/Server/project/assets/database/locales/global/en.json"
+    )
+
+    print("a")
+
+    # [
+    #     print(ti["_props"]["Caliber"])
+    #     for tpl, ti in tarkov_items._items.items()
+    #     if tarkov_items.is_baseclass(tpl,tarkov_items.BASECLASS.Ammo)
+    # ]
+
+    for tpl, ti in tarkov_items._items.items():
+        if tarkov_items.is_baseclass(tpl, tarkov_items.BASECLASS.Magazine):
+            try:
+                ammoTpls = ti["_props"]["Cartridges"][0]["_props"]["filters"][0]["Filter"]
+            except Exception as e:
+                break
+
+            ammos = [tarkov_items.template(tpli) for tpli in ammoTpls]
+            for ai in ammos:
+                try:
+                    test = ai["_props"]["Caliber"]
+                    print(test)
+                except Exception as e:
+                    print(e)
+                    print(ai)
+
+    # print(tarkov_items.template_locale_name("59e6152586f77473dc057aa1"))
+    #
+    # print(tarkov_items.price("59e6152586f77473dc057aa1"))
+    # print(tarkov_items.ancestry("59e6152586f77473dc057aa1"))
+    # print(tarkov_items.is_baseclass("59e6152586f77473dc057aa1", TarkovItems.BASECLASS.Weapon))
+    #
+    # print(tarkov_items.ancestry(TarkovItems.BASECLASS.Armor))
+    # print(tarkov_items.ancestry(TarkovItems.BASECLASS.Headwear))
+    # print(tarkov_items.ancestry(TarkovItems.BASECLASS.Vest))