From effbfe3462f1538fcda179976e744ff0484d178e Mon Sep 17 00:00:00 2001 From: Kurt Date: Fri, 12 Apr 2024 23:10:10 -0700 Subject: [PATCH 01/15] Update activity list to include Iron Banner Rift and Iron Banner Zone Control. Update reports to include new activities. --- app/data/activities.py | 5 +++-- app/reports/ActivityCountReport.py | 2 +- app/reports/ActivityLocationTimeReport.py | 2 +- app/reports/ActivityLocationWeaponReport.py | 2 +- app/reports/ActivityWinrateReport.py | 2 +- app/reports/KDReport.py | 2 +- app/reports/KillsDeathsAssistsReport.py | 2 +- app/reports/WeaponRaceReport.py | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/data/activities.py b/app/data/activities.py index 171b830..095ed04 100644 --- a/app/data/activities.py +++ b/app/data/activities.py @@ -58,8 +58,9 @@ 87: 'Lost Sector', 88: "Rift", 89: "Zone Control", - 90: "Iron Banner Rift" + 90: "Iron Banner Rift", + 91: "Iron Banner Zone Control" } -PVP_ACTIVITIES = [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] +PVP_ACTIVITIES = [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] GAMBIT_ACTIVITIES = [75, 63] diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index b293d41..01d4fbd 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -39,7 +39,7 @@ def generateData(self, data): for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue typus = "PvE" - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index bc79f03..7866d64 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -46,7 +46,7 @@ def generateData(self, data): start_date = datetime.fromtimestamp(timestamp + starts) typus = "PvE" - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index 4bc5db1..57baaf7 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -47,7 +47,7 @@ def generateData(self, data): for wp in entry["extended"]["weapons"]: typus = "PvE" - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index 77b45fb..c6850f3 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -41,7 +41,7 @@ def generateData(self, data): timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index dc6106a..87222cc 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -80,7 +80,7 @@ def generateRawDataframe(self, data): "deaths": deaths, "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 return df diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index 699c3ef..d39e71b 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -78,7 +78,7 @@ def generateRawDataframe(self, data): "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 df['Date'] = pd.to_datetime(df['start']) - pd.to_timedelta(7, unit='d') return df diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index a1b7ff1..bce993a 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -66,7 +66,7 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit if "weapons" not in entry["extended"]: continue typus = "pve" - if data["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if data["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "pvp" #elif data["activityDetails"]["mode"] in [75, 63]: # typus = "gambit" if typus != typ: From 9b20895cb96f682002995c241704025232a440fd Mon Sep 17 00:00:00 2001 From: Kurt Date: Sat, 13 Apr 2024 13:47:26 -0700 Subject: [PATCH 02/15] Create .gitignore --- .gitignore | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9ba3c69 --- /dev/null +++ b/.gitignore @@ -0,0 +1,183 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python + +# ignore generated output data +/data +/cache + +# visual studio remnants +/.vscode From 19c61a60e24351e6d9f191dfd54a7d372a44cab0 Mon Sep 17 00:00:00 2001 From: Kurt Date: Sat, 13 Apr 2024 13:49:52 -0700 Subject: [PATCH 03/15] Create cache folder for manifest contents. Includes version check. --- app/bungiemanifest.py | 62 +++++++++++++++++++-- app/data/activities.py | 2 +- app/reports/ActivityCountReport.py | 2 +- app/reports/ActivityLocationTimeReport.py | 2 +- app/reports/ActivityLocationWeaponReport.py | 2 +- app/reports/ActivityWinrateReport.py | 2 +- app/reports/KDReport.py | 2 +- app/reports/KillsDeathsAssistsReport.py | 2 +- app/reports/WeaponRaceReport.py | 2 +- main.py | 19 +++++-- 10 files changed, 81 insertions(+), 16 deletions(-) diff --git a/app/bungiemanifest.py b/app/bungiemanifest.py index 0a68d41..edddf25 100644 --- a/app/bungiemanifest.py +++ b/app/bungiemanifest.py @@ -1,27 +1,79 @@ -import json, urllib.request +import json, urllib.request, os, shutil from app.data.activities import ACTIVITY_NAMES BUNGIE_BASE = "https://bungie.net/" BUNGIE_API_BASE = "https://bungie.net/Platform/" - class DestinyManifest(): def __init__(self): self.ActivityNames = None self.ActivityTypeNames = None self.ItemDefinitions = None + self.CacheFolder = None def update(self): + self.CacheFolder = GetCacheFolder() self.ActivityTypeNames = GetActivityTypeNames() + self.VersionNumber = GetVersionNumber() self.ItemDefinitions = GetInventoryItemDefinitions() self.ActivityNames = GetActivityNames() return self - + +def GetCacheFolder(): + # get current file path and go up two directories + path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + path = os.path.join(path, 'cache') + os.makedirs(path, exist_ok=True) + return path + +def GetVersionNumber(): + print("Check version number") + # get version from manifest file + manifestPaths = json.loads(urllib.request.urlopen(BUNGIE_API_BASE + "/Destiny2/Manifest/").read())["Response"] + version = manifestPaths['version'] + cacheRoot = GetCacheFolder() + + # if version file exists and matches response, continue + if os.path.exists(os.path.join(cacheRoot, version)): + print("Version check passed") + return + + # if version file does not exist or does not match + print("Version check failed, deleting cache") + # delete existing cache folder + shutil.rmtree(cacheRoot, ignore_errors=True) + print("Updating version") + # create cache folder + versionFilePath = os.path.join(cacheRoot, version) + # create new version file + with open(file=versionFilePath, mode='w', encoding='utf-8') as f: + f.write(version) + +def SaveToCache(aDefinition, aJsonBlob): + # set root path + filePath = os.path.join(GetCacheFolder(), aDefinition) + with open(file=filePath, mode='w', encoding='utf-8') as f: + json.dump(aJsonBlob, f, ensure_ascii=False, indent=4) + +def LoadFromCache(aDefinition): + filePath = os.path.join(GetCacheFolder(), aDefinition) + exists = os.path.exists(filePath) + if not exists: + return None, False + with open(file=filePath, mode='r', encoding='utf-8') as f: + blob = json.load(f) + return blob, True def GetManifestDefinitions(definition): print("Get %s" % definition) + print("Check %s saved in cache" % definition) + blob, exists = LoadFromCache(definition) + if exists: + print("Found %s in cache" % definition) + return blob + manifestPaths = json.loads(urllib.request.urlopen(BUNGIE_API_BASE + "/Destiny2/Manifest/").read())["Response"] manifestPath = manifestPaths["jsonWorldComponentContentPaths"]["en"][definition] print("Get %s from '%s'" % (definition, BUNGIE_BASE + manifestPath)) @@ -29,7 +81,9 @@ def GetManifestDefinitions(definition): print("Unpack and parse %s" % definition) InventoryItemDefinitions = json.loads(InventoryItemDefinitions) - print("Json'd %s" % definition) + print("Json'd %s and cache'd" % definition) + SaveToCache(definition, InventoryItemDefinitions) + return InventoryItemDefinitions diff --git a/app/data/activities.py b/app/data/activities.py index 095ed04..7d80c4d 100644 --- a/app/data/activities.py +++ b/app/data/activities.py @@ -62,5 +62,5 @@ 91: "Iron Banner Zone Control" } -PVP_ACTIVITIES = [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] +PVP_ACTIVITIES = [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] GAMBIT_ACTIVITIES = [75, 63] diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index 01d4fbd..4cce29b 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -39,7 +39,7 @@ def generateData(self, data): for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue typus = "PvE" - if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index 7866d64..a2015ef 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -46,7 +46,7 @@ def generateData(self, data): start_date = datetime.fromtimestamp(timestamp + starts) typus = "PvE" - if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index 57baaf7..b2543eb 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -47,7 +47,7 @@ def generateData(self, data): for wp in entry["extended"]["weapons"]: typus = "PvE" - if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index c6850f3..e22e8ca 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -41,7 +41,7 @@ def generateData(self, data): timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue - if datapoint["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index 87222cc..4f77d00 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -80,7 +80,7 @@ def generateRawDataframe(self, data): "deaths": deaths, "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 return df diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index d39e71b..3632cb2 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -78,7 +78,7 @@ def generateRawDataframe(self, data): "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 df['Date'] = pd.to_datetime(df['start']) - pd.to_timedelta(7, unit='d') return df diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index bce993a..9f8d219 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -66,7 +66,7 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit if "weapons" not in entry["extended"]: continue typus = "pve" - if data["activityDetails"]["mode"] in [91, 90, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if data["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "pvp" #elif data["activityDetails"]["mode"] in [75, 63]: # typus = "gambit" if typus != typ: diff --git a/main.py b/main.py index f2e1c26..be67845 100644 --- a/main.py +++ b/main.py @@ -20,6 +20,7 @@ from app.reports.WeaponReport import WeaponReport from app.reports.WeekdayReport import WeekdayReport # from app.DiscordSender import DiscordSender +import os if __name__ == '__main__': import pathos @@ -27,15 +28,25 @@ pathos.helpers.freeze_support() # required for windows manifest = DestinyManifest().update() - pool = ProcessPool(25) + pool = ProcessPool(5) # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process, but takes serious computation power # pool = ProcessPool(60) + # Xbox 1 + # Psn 2 + # Steam 3 + # Blizzard 4 + # Stadia 5 + # Egs 6 + # MEMBERSHIP_NAME = (platform, bungie ID) + MEMBERSHIP_MIJAGO = (3, 4611686018482684809) - USED_MEMBERSHIP = MEMBERSHIP_MIJAGO + MEMBERSHIP_SUPERQ = (3, 4611686018472661350) + USED_MEMBERSHIP = MEMBERSHIP_SUPERQ - api = BungieApi("API-KEY") - VIDEO_TYPE = "gif" # you can also use "mp4" if you installed ffmpeg; see README.d + api = BungieApi(os.getenv('api_key')) + # "gif by default, "mp4" if you installed ffmpeg; see README.d + VIDEO_TYPE = "mp4" Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) Director.ClearResultDirectory(*USED_MEMBERSHIP) From 9c6daa96887e49decf5b60f3beb30810fc2a628a Mon Sep 17 00:00:00 2001 From: Kurt Date: Sat, 13 Apr 2024 13:54:17 -0700 Subject: [PATCH 04/15] Need to recreate cache folder after deleting it to save the new version file properly. --- app/bungiemanifest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/bungiemanifest.py b/app/bungiemanifest.py index edddf25..c02c285 100644 --- a/app/bungiemanifest.py +++ b/app/bungiemanifest.py @@ -46,6 +46,7 @@ def GetVersionNumber(): shutil.rmtree(cacheRoot, ignore_errors=True) print("Updating version") # create cache folder + GetCacheFolder() versionFilePath = os.path.join(cacheRoot, version) # create new version file with open(file=versionFilePath, mode='w', encoding='utf-8') as f: From ea5d937e072b856810acaeb06e62a8023a554707 Mon Sep 17 00:00:00 2001 From: Kurt Date: Sun, 14 Apr 2024 15:14:55 -0700 Subject: [PATCH 05/15] Add new PvP mode Relic to activities and reports. --- app/data/activities.py | 5 +++-- app/reports/ActivityCountReport.py | 5 +++-- app/reports/ActivityLocationTimeReport.py | 2 +- app/reports/ActivityLocationWeaponReport.py | 2 +- app/reports/ActivityWinrateReport.py | 2 +- app/reports/KDReport.py | 2 +- app/reports/KillsDeathsAssistsReport.py | 2 +- app/reports/WeaponRaceReport.py | 6 +++--- 8 files changed, 14 insertions(+), 12 deletions(-) diff --git a/app/data/activities.py b/app/data/activities.py index 7d80c4d..690f893 100644 --- a/app/data/activities.py +++ b/app/data/activities.py @@ -59,8 +59,9 @@ 88: "Rift", 89: "Zone Control", 90: "Iron Banner Rift", - 91: "Iron Banner Zone Control" + 91: "Iron Banner Zone Control", + 92: "Relic" } -PVP_ACTIVITIES = [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] +PVP_ACTIVITIES = [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] GAMBIT_ACTIVITIES = [75, 63] diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index 4cce29b..ef826ab 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -39,7 +39,7 @@ def generateData(self, data): for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue typus = "PvE" - if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" @@ -62,4 +62,5 @@ def generateData(self, data): }) df = df.groupby(["type", "mode", "directorActivity"]).size().reset_index(name='count') - return df \ No newline at end of file + return df + \ No newline at end of file diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index a2015ef..e4b226c 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -46,7 +46,7 @@ def generateData(self, data): start_date = datetime.fromtimestamp(timestamp + starts) typus = "PvE" - if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index b2543eb..74b9500 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -47,7 +47,7 @@ def generateData(self, data): for wp in entry["extended"]["weapons"]: typus = "PvE" - if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index e22e8ca..51fb0c3 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -41,7 +41,7 @@ def generateData(self, data): timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue - if datapoint["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index 4f77d00..480b9be 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -80,7 +80,7 @@ def generateRawDataframe(self, data): "deaths": deaths, "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 return df diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index 3632cb2..2e09600 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -78,7 +78,7 @@ def generateRawDataframe(self, data): "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 df['Date'] = pd.to_datetime(df['start']) - pd.to_timedelta(7, unit='d') return df diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index 9f8d219..a2ca7bb 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -66,7 +66,7 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit if "weapons" not in entry["extended"]: continue typus = "pve" - if data["activityDetails"]["mode"] in [91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if data["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "pvp" #elif data["activityDetails"]["mode"] in [75, 63]: # typus = "gambit" if typus != typ: @@ -85,8 +85,8 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit ) df = pd.DataFrame(eps, columns=["date", "name", "kills"]) - df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum().reset_index() - df2["cumsum"] = df2.groupby(["name"]).cumsum() + df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum(numeric_only=True).reset_index() + df2["cumsum"] = df2.groupby(["name"]).cumsum(numeric_only=True) df3 = df2.pivot(index="date", columns="name", values="cumsum") df3 = df3.fillna(method='ffill') From 82e8a769e6566ff46313ff32ac5561e62cc08d42 Mon Sep 17 00:00:00 2001 From: Kurt Date: Sun, 14 Apr 2024 15:16:28 -0700 Subject: [PATCH 06/15] Updates to run Python 3.12 and Pandas 2.2.2 --- app/PgcrCollector.py | 9 ++++++--- app/reports/ActivityTypeRaceReport.py | 4 ++-- app/reports/FireteamActivityReport.py | 2 +- app/reports/FireteamRace.py | 6 +++--- app/reports/WeaponReport.py | 2 +- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index 01c9adb..2a5dc8c 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -78,7 +78,7 @@ def downloadPGCR(activity): tries += 1 pgcr = bungo.getPGCR(id) - with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.membershipType, self.membershipId), pgcr["activityDetails"]["instanceId"]), "w") as f: + with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.membershipType, self.membershipId), pgcr["activityDetails"]["instanceId"]), "w", encoding='utf-8') as f: f.write(json.dumps(pgcr)) stepsize = pagesize @@ -111,8 +111,11 @@ def loadJson(fnameList): for fname in fnameList: if fname is None: continue - with open(fname, "r") as f: - r.append( json.load(f)) + with open(fname, "r", encoding='utf-8') as f: + try: + r.append(json.load(f)) + except Exception: + print('Error on %s' % fname) return r with Timer("Get all PGCRs from individual files"): diff --git a/app/reports/ActivityTypeRaceReport.py b/app/reports/ActivityTypeRaceReport.py index 52d617d..e57dee4 100644 --- a/app/reports/ActivityTypeRaceReport.py +++ b/app/reports/ActivityTypeRaceReport.py @@ -67,8 +67,8 @@ def generateData(self, datap): # pve, pvp, gambit ) df = pd.DataFrame(eps, columns=["date", "name", "hours"]) - df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum().reset_index() - df2["cumsum"] = df2.groupby(["name"]).cumsum() + df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum(numeric_only=True).reset_index() + df2["cumsum"] = df2.groupby(["name"]).cumsum(numeric_only=True) df3 = df2.pivot(index="date", columns="name", values="cumsum") df3 = df3.fillna(method='ffill') diff --git a/app/reports/FireteamActivityReport.py b/app/reports/FireteamActivityReport.py index 3c8df5b..9af7394 100644 --- a/app/reports/FireteamActivityReport.py +++ b/app/reports/FireteamActivityReport.py @@ -8,7 +8,7 @@ class FireteamActivityReport(Report): def save(self): with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - fireteam member activities"), "w", encoding='utf-8') as f: - self.df.to_csv(f, index=False, line_terminator='\n') + self.df.to_csv(f, index=False) print("Report> Generated %s" % self.getName()) def getName(self) -> str: diff --git a/app/reports/FireteamRace.py b/app/reports/FireteamRace.py index 496187f..b019add 100644 --- a/app/reports/FireteamRace.py +++ b/app/reports/FireteamRace.py @@ -89,9 +89,9 @@ def generateData(self, datap): eps += res - df = pd.DataFrame(eps, columns=["date", "name", "minutes"]) - df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum().reset_index() - df2["cumsum"] = df2.groupby(["name"]).cumsum() + df = pd.DataFrame(data=eps, columns=["date", "name", "minutes"]) + df2 = df.groupby(by=[df.date.dt.to_period(freq='W'), "name"]).sum(numeric_only=True).reset_index() + df2["cumsum"] = df2.groupby(["name"]).cumsum(numeric_only=True) df3 = df2.pivot(index="date", columns="name", values="cumsum") df3 = df3.fillna(method='ffill') diff --git a/app/reports/WeaponReport.py b/app/reports/WeaponReport.py index 7976d1b..84f52fe 100644 --- a/app/reports/WeaponReport.py +++ b/app/reports/WeaponReport.py @@ -19,7 +19,7 @@ def save(self): "
Generated by Mijago") f.write(build_table(self.df, 'blue_light')) with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - weapons used per activity type"), "w") as f: - self.df.to_csv(f, index=False, line_terminator='\n') + self.df.to_csv(f, index=False) super().save() From 035e09685d873368e210329644a1373c3bd68b33 Mon Sep 17 00:00:00 2001 From: Kurt Date: Sun, 14 Apr 2024 15:17:42 -0700 Subject: [PATCH 07/15] Updates --- main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index be67845..9b833cc 100644 --- a/main.py +++ b/main.py @@ -28,8 +28,8 @@ pathos.helpers.freeze_support() # required for windows manifest = DestinyManifest().update() - pool = ProcessPool(5) - # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process, but takes serious computation power + pool = ProcessPool() + # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process but takes serious computation power. # pool = ProcessPool(60) # Xbox 1 @@ -42,7 +42,9 @@ MEMBERSHIP_MIJAGO = (3, 4611686018482684809) MEMBERSHIP_SUPERQ = (3, 4611686018472661350) - USED_MEMBERSHIP = MEMBERSHIP_SUPERQ + MEMBERSHIP_SHTGUNWEDDING = (2, 4611686018428655241) + + USED_MEMBERSHIP = MEMBERSHIP_SHTGUNWEDDING api = BungieApi(os.getenv('api_key')) # "gif by default, "mp4" if you installed ffmpeg; see README.d From 77da5803b58b095a59470a85edeee9c95b469a0f Mon Sep 17 00:00:00 2001 From: Kurt Date: Wed, 17 Apr 2024 00:35:10 -0700 Subject: [PATCH 08/15] Add argument parsing --- main.py | 50 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/main.py b/main.py index 9b833cc..081ea43 100644 --- a/main.py +++ b/main.py @@ -20,34 +20,48 @@ from app.reports.WeaponReport import WeaponReport from app.reports.WeekdayReport import WeekdayReport # from app.DiscordSender import DiscordSender -import os +############################################################################### +# +# main() +# +############################################################################### if __name__ == '__main__': - import pathos + import pathos, argparse, os + + # build argument parsing + descriptionString = """Get and compile stats for a Destiny 2 user. + example: main.py 3 4611686018482684809""" + platformString = """ Xbox 1 + Psn 2 + Steam 3 + Blizzard 4 + Stadia 5 + Egs 6""" + parser = argparse.ArgumentParser(prog='main.py', description=f'{descriptionString}', formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('-p', type=int, required=False, dest='Platform', help=f'{platformString}') + parser.add_argument('-id', type=int, required=False, help='bungie ID') + args = parser.parse_args() + from pathos.multiprocessing import ProcessPool pathos.helpers.freeze_support() # required for windows - manifest = DestinyManifest().update() - pool = ProcessPool() # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process but takes serious computation power. # pool = ProcessPool(60) - # Xbox 1 - # Psn 2 - # Steam 3 - # Blizzard 4 - # Stadia 5 - # Egs 6 - # MEMBERSHIP_NAME = (platform, bungie ID) + MIJAGO = (3, 4611686018482684809) + SUPERQ = (3, 4611686018472661350) + SHTGUNWEDDING = (2, 4611686018428655241) + EURO = (3, 4611686018471254627) + DREDGENQ = (3, 4611686018534347056) + SPRQMAN = (2, 4611686018436271063) + USED_MEMBERSHIP = SHTGUNWEDDING - MEMBERSHIP_MIJAGO = (3, 4611686018482684809) - MEMBERSHIP_SUPERQ = (3, 4611686018472661350) - MEMBERSHIP_SHTGUNWEDDING = (2, 4611686018428655241) - - USED_MEMBERSHIP = MEMBERSHIP_SHTGUNWEDDING + # check manifest + manifest = DestinyManifest().update() - api = BungieApi(os.getenv('api_key')) - # "gif by default, "mp4" if you installed ffmpeg; see README.d + api = BungieApi(os.getenv('BUNGIE_API_KEY')) + # "gif by default, "mp4" if you installed ffmpeg which you should; see README.d VIDEO_TYPE = "mp4" Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) From 3c6012a23baf50eaa596f6a47bc8d8ac1887c86a Mon Sep 17 00:00:00 2001 From: Kurt Date: Thu, 18 Apr 2024 00:11:12 -0700 Subject: [PATCH 09/15] Improve report generation logging --- app/reports/ActivityCountReport.py | 3 +- app/reports/ActivityLocationTimeReport.py | 3 +- app/reports/ActivityLocationWeaponReport.py | 3 +- app/reports/ActivityTypeRaceReport.py | 6 ++- app/reports/ActivityWinrateReport.py | 3 +- app/reports/FireteamActivityReport.py | 6 ++- app/reports/FireteamRace.py | 4 +- app/reports/KDReport.py | 4 +- app/reports/KillsDeathsAssistsReport.py | 4 +- app/reports/LightLevelReport.py | 4 +- app/reports/PlaytimeCharacterReport.py | 3 +- app/reports/PlaytimeReport.py | 4 +- app/reports/ReportBase.py | 3 +- app/reports/WeaponKillTreeReport.py | 4 +- app/reports/WeaponRaceReport.py | 4 +- app/reports/WeaponReport.py | 4 +- app/reports/WeekdayReport.py | 4 +- main.py | 45 ++++++++++++--------- 18 files changed, 72 insertions(+), 39 deletions(-) diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index ef826ab..0d90417 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -29,12 +29,13 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm typ = [] mode = [] directorActivity = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index e4b226c..6c74fd4 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -29,12 +29,13 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm category = [] playtime = [] activity = [] directorActivity = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index 74b9500..5ee7f35 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -30,6 +30,7 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm category = [] kills = [] @@ -37,7 +38,7 @@ def generateData(self, data): weapon = [] directorActivity = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/ActivityTypeRaceReport.py b/app/reports/ActivityTypeRaceReport.py index e57dee4..4377aca 100644 --- a/app/reports/ActivityTypeRaceReport.py +++ b/app/reports/ActivityTypeRaceReport.py @@ -50,9 +50,13 @@ def generate(self, data): return self def generateData(self, datap): # pve, pvp, gambit + from tqdm import tqdm + import warnings + warnings.simplefilter("ignore") + eps = [] - for data in datap: + for data in tqdm(datap): if "entries" not in data: continue date = parser.parse(data["period"]) # find own user entry diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index 51fb0c3..4092c36 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -30,13 +30,14 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm typ = [] mode = [] directorActivity = [] wintype = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/FireteamActivityReport.py b/app/reports/FireteamActivityReport.py index 9af7394..44fd208 100644 --- a/app/reports/FireteamActivityReport.py +++ b/app/reports/FireteamActivityReport.py @@ -9,7 +9,6 @@ class FireteamActivityReport(Report): def save(self): with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - fireteam member activities"), "w", encoding='utf-8') as f: self.df.to_csv(f, index=False) - print("Report> Generated %s" % self.getName()) def getName(self) -> str: return "[ALL] table - fireteam member activities" @@ -23,10 +22,13 @@ def generate(self, data) -> Report: return self def generateListDataframe(self, datap): + from tqdm import tqdm + eps = [] displayNames = dict() displayNameTimes = dict() - for data in datap: + + for data in tqdm(datap): if "entries" not in data: continue # find own user entry entry = [e for e in data["entries"] if e["player"]["destinyUserInfo"]["membershipId"] == str(self.membershipId)][0] diff --git a/app/reports/FireteamRace.py b/app/reports/FireteamRace.py index b019add..c3e6462 100644 --- a/app/reports/FireteamRace.py +++ b/app/reports/FireteamRace.py @@ -49,11 +49,13 @@ def generate(self, data) -> Report: return self def generateData(self, datap): + from tqdm import tqdm + eps = [] displayNames = dict() displayNameTimes = dict() - for data in datap: + for data in tqdm(datap): if "entries" not in data: continue # find own user entry entry = [e for e in data["entries"] if e["player"]["destinyUserInfo"]["membershipId"] == str(self.membershipId)][0] diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index 480b9be..0cf7545 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -39,6 +39,8 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] starttime_str = [] endtime_str = [] @@ -49,7 +51,7 @@ def generateRawDataframe(self, data): deaths = [] kills = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index 2e09600..142e207 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -40,6 +40,8 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] starttime_str = [] endtime_str = [] @@ -49,7 +51,7 @@ def generateRawDataframe(self, data): deaths = [] kills = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/LightLevelReport.py b/app/reports/LightLevelReport.py index f0b842e..1538d07 100644 --- a/app/reports/LightLevelReport.py +++ b/app/reports/LightLevelReport.py @@ -36,11 +36,13 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] endtime = [] lightlevel = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/PlaytimeCharacterReport.py b/app/reports/PlaytimeCharacterReport.py index 24d675d..0f5091d 100644 --- a/app/reports/PlaytimeCharacterReport.py +++ b/app/reports/PlaytimeCharacterReport.py @@ -40,6 +40,7 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm starttime_str = [] starttime = [] @@ -47,7 +48,7 @@ def generateRawDataframe(self, data): clazz = [] character = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/PlaytimeReport.py b/app/reports/PlaytimeReport.py index 77e465a..cccd29b 100644 --- a/app/reports/PlaytimeReport.py +++ b/app/reports/PlaytimeReport.py @@ -35,12 +35,14 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] endtime = [] playtime = [] mode = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/ReportBase.py b/app/reports/ReportBase.py index 3fb132d..3f2766e 100644 --- a/app/reports/ReportBase.py +++ b/app/reports/ReportBase.py @@ -22,9 +22,8 @@ def getName(self) -> str: return "unnamed" def save(self): + print("Report> Saving %s" % self.getName()) assert self.fig is not None filename = '%s/%s.html' % (Director.GetResultDirectory(self.membershipType, self.membershipId), self.getName()) with open(filename, "w") as f: f.write(self.fig.to_html(full_html=False, include_plotlyjs='cdn')) - - print("Report> Generated %s" % self.getName()) diff --git a/app/reports/WeaponKillTreeReport.py b/app/reports/WeaponKillTreeReport.py index 6c6aa4b..2dcee35 100644 --- a/app/reports/WeaponKillTreeReport.py +++ b/app/reports/WeaponKillTreeReport.py @@ -27,12 +27,14 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm + weaponType = [] weapon = [] killType = [] kills = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index a2ca7bb..87a88b3 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -56,9 +56,11 @@ def generateintern(self, data, type): ) def generateData(self, datap, typ="pve"): # pve, pvp, gambit + from tqdm import tqdm + eps = [] - for data in datap: + for data in tqdm(datap): if "entries" not in data: continue date = parser.parse(data["period"]) # find own user entry diff --git a/app/reports/WeaponReport.py b/app/reports/WeaponReport.py index 84f52fe..e33b6c3 100644 --- a/app/reports/WeaponReport.py +++ b/app/reports/WeaponReport.py @@ -51,6 +51,8 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] endtime = [] weapon = [] @@ -62,7 +64,7 @@ def generateRawDataframe(self, data): mode_name = [] kills_precision = [] - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/WeekdayReport.py b/app/reports/WeekdayReport.py index e7f55bd..affc78d 100644 --- a/app/reports/WeekdayReport.py +++ b/app/reports/WeekdayReport.py @@ -43,6 +43,8 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm + timeArray = [[0 for k in range(0, 24)] for n in range(0, 7)] lookup = set() @@ -54,7 +56,7 @@ def generateKey(xdate): xdate.hour ) - for datapoint in data: + for datapoint in tqdm(data): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/main.py b/main.py index 081ea43..868fd1e 100644 --- a/main.py +++ b/main.py @@ -39,9 +39,22 @@ Stadia 5 Egs 6""" parser = argparse.ArgumentParser(prog='main.py', description=f'{descriptionString}', formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('-p', type=int, required=False, dest='Platform', help=f'{platformString}') + parser.add_argument('-p', type=int, required=False, dest='platform', help=f'{platformString}') parser.add_argument('-id', type=int, required=False, help='bungie ID') - args = parser.parse_args() + args = vars(parser.parse_args()) + platform = args['platform'] + id = args['id'] + + if platform != None and id != None: + USED_MEMBERSHIP = (platform, id) + else: + MIJAGO = (3, 4611686018482684809) + SUPERQ = (3, 4611686018472661350) + SHTGUNWEDDING = (2, 4611686018428655241) + EURO = (3, 4611686018471254627) + DREDGENQ = (3, 4611686018534347056) + SPRQMAN = (2, 4611686018436271063) + USED_MEMBERSHIP = DREDGENQ from pathos.multiprocessing import ProcessPool pathos.helpers.freeze_support() # required for windows @@ -49,14 +62,6 @@ # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process but takes serious computation power. # pool = ProcessPool(60) - MIJAGO = (3, 4611686018482684809) - SUPERQ = (3, 4611686018472661350) - SHTGUNWEDDING = (2, 4611686018428655241) - EURO = (3, 4611686018471254627) - DREDGENQ = (3, 4611686018534347056) - SPRQMAN = (2, 4611686018436271063) - USED_MEMBERSHIP = SHTGUNWEDDING - # check manifest manifest = DestinyManifest().update() @@ -74,22 +79,22 @@ pool.close() reports = [ - KDReport(*USED_MEMBERSHIP, manifest), - KillsDeathsAssistsReport(*USED_MEMBERSHIP, manifest), - WeaponReport(*USED_MEMBERSHIP, manifest), - LightLevelReport(*USED_MEMBERSHIP, manifest), - PlaytimeReport(*USED_MEMBERSHIP, manifest), - PlaytimeCharacterReport(*USED_MEMBERSHIP, manifest), ActivityCountReport(*USED_MEMBERSHIP, manifest), - WeekdayReport(*USED_MEMBERSHIP, manifest), ActivityLocationTimeReport(*USED_MEMBERSHIP, manifest), ActivityLocationWeaponReport(*USED_MEMBERSHIP, manifest), + ActivityTypeRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), ActivityWinrateReport(*USED_MEMBERSHIP, manifest), - WeaponKillTreeReport(*USED_MEMBERSHIP, manifest), + FireteamActivityReport(*USED_MEMBERSHIP, manifest), FireteamRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), + KDReport(*USED_MEMBERSHIP, manifest), + KillsDeathsAssistsReport(*USED_MEMBERSHIP, manifest), + LightLevelReport(*USED_MEMBERSHIP, manifest), + PlaytimeCharacterReport(*USED_MEMBERSHIP, manifest), + PlaytimeReport(*USED_MEMBERSHIP, manifest), + WeaponKillTreeReport(*USED_MEMBERSHIP, manifest), WeaponRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - ActivityTypeRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - FireteamActivityReport(*USED_MEMBERSHIP, manifest) + WeaponReport(*USED_MEMBERSHIP, manifest), + WeekdayReport(*USED_MEMBERSHIP, manifest) ] for report in reports: report.generate(data).save() From 430a068d6b147f13d084dc84df65287a15491a4e Mon Sep 17 00:00:00 2001 From: Mijago Date: Fri, 19 Apr 2024 14:48:55 +0000 Subject: [PATCH 10/15] Minor improvements, Readme updates --- README.md | 16 +++++++++------- app/DiscordSender.py | 17 ----------------- app/PgcrCollector.py | 14 +++----------- main.py | 35 ++++++++++++++++------------------- 4 files changed, 28 insertions(+), 54 deletions(-) delete mode 100644 app/DiscordSender.py diff --git a/README.md b/README.md index ef436f1..f5d1494 100644 --- a/README.md +++ b/README.md @@ -9,22 +9,24 @@ My Twitter: https://twitter.com/MijagoCoding/ # How to Use? 3) Install all required packages - 1) `python -m pip install pandas plotly pathos requests pretty_html_table bar-chart-race` - 2) If you want to use mp4 instead of gif, also install `python -m pip install python-ffmpeg` and put a [ffmpeg](https://www.ffmpeg.org/download.html) in your PATH variable. Then set the `VIDEO_TYPE` in `main.py` to `mp4`. + 1) `python3 -m pip install pandas plotly pathos requests pretty_html_table bar-chart-race tqdm` + 2) If you want to use mp4 instead of gif, also install `python3 -m pip install python-ffmpeg` and put a [ffmpeg](https://www.ffmpeg.org/download.html) in your PATH variable. Then set the `VIDEO_TYPE` in `main.py` to `mp4`. **I highly encourage you to do this, as the gifs tend to be 40mb in size, whereas the mp4 is only around 1.5mb~2mb**. Download it here: [ffmpeg](https://github.com/BtbN/FFmpeg-Builds/releases) (for Windows, `ffmpeg-n5.0-latest-win64-gpl-5.0.zip`). -4) Add your api key to `main.py`. For this, edit `api = BungieApi("API-KEY")`. Get it [here](https://www.bungie.net/en/Application). -5) Edit your user info in `main.py`. +4) Set your API key as an environemnt variable `BUNGIE_API_KEY`. Get the key [here](https://www.bungie.net/en/Application). + 1) Alternatively: Add your api key to `main.py`. For this, edit `# API_KEY = "123456789"`. +5) Edit your user info in `main.py`. Alternatively, you can also use command line parameters to set this later. ```py MEMBERSHIP_MIJAGO = (3, 4611686018482684809) MEMBERSHIP_MYCOOLID = (1, 1231231231231233353) # for example, add this USED_MEMBERSHIP = MEMBERSHIP_MYCOOLID - api = BungieApi("API-KEY") ``` -6) Run! `python3 main.py` - 1) May take a while. I need 35~45 seconds for 1000 PGCRs with a download speed of 4.5mb/s. +6) Run the script `python3 main.py`. + 1) Complete Example: `BUNGIE_API_KEY=123456789012345 python3 main.py -p 3 -id 4611686018482684809` + 2) Alternatively you can also specify the platform and user: `python3 main.py -p 3 -id 4611686018482684809` + 3) This may take a while. I need 35~45 seconds for 1000 PGCRs with a download speed of 4.5mb/s. # Where do I get my user ID? 1) Go to https://www.d2checklist.com (or any other similar page) diff --git a/app/DiscordSender.py b/app/DiscordSender.py deleted file mode 100644 index 47f05fb..0000000 --- a/app/DiscordSender.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -import discord - -from app.Director import Director - - -class DiscordSender: - @staticmethod - def send(zipPath, membershipType, membershipId): - # DISCORD WEBHOOK - webhook = discord.Webhook.partial(935137211285802, 'q64vyCdgFKCdq5pdhfoU95UEKD7vwolUSAb7SKFJyX5iP3pHWJ5G964fp7s3xDlRb', adapter=discord.RequestsWebhookAdapter()) # Your webhook - - with open(zipPath, "rb") as f: - zipFile = discord.File(f, filename="Report_%d_%d.zip" % (membershipType, membershipId)) - - webhook.send("Report for %d_%d" % (membershipType, membershipId), username="Mijago's PgcrReport Generator", file=zipFile) diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index 2a5dc8c..842013b 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -66,7 +66,7 @@ def downloadActivityPage(page): return self - def getPGCRs(self, pagesize=1000): + def getPGCRs(self): bungo = self.api def downloadPGCR(activity): @@ -81,20 +81,12 @@ def downloadPGCR(activity): with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.membershipType, self.membershipId), pgcr["activityDetails"]["instanceId"]), "w", encoding='utf-8') as f: f.write(json.dumps(pgcr)) - stepsize = pagesize - START_PAGE = 0 - if len(self.activities) == 0: print("No activities to grab") return self - for steps in range(START_PAGE, (len(self.activities) + stepsize - 1) // stepsize): - try: - with Timer("Get PGCRs %d through %d" % (steps * stepsize + 1, min(len(self.activities), (steps + 1) * stepsize))): - # self.processPool.restart(True) - self.processPool.amap(downloadPGCR, self.activities[steps * stepsize:(steps + 1) * stepsize]).get() - except Exception as e: - print(e) + from tqdm.auto import tqdm + list(tqdm(self.processPool.imap(downloadPGCR, self.activities), total=len(self.activities), desc="Downloading PGCRs")) return self def combineAllPgcrs(self): diff --git a/main.py b/main.py index 868fd1e..5a901fa 100644 --- a/main.py +++ b/main.py @@ -19,7 +19,6 @@ from app.reports.WeaponRaceReport import WeaponRaceReport from app.reports.WeaponReport import WeaponReport from app.reports.WeekdayReport import WeekdayReport -# from app.DiscordSender import DiscordSender ############################################################################### # @@ -31,7 +30,7 @@ # build argument parsing descriptionString = """Get and compile stats for a Destiny 2 user. - example: main.py 3 4611686018482684809""" + example: main.py -p 3 -id 4611686018482684809""" platformString = """ Xbox 1 Psn 2 Steam 3 @@ -39,33 +38,34 @@ Stadia 5 Egs 6""" parser = argparse.ArgumentParser(prog='main.py', description=f'{descriptionString}', formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('-p', type=int, required=False, dest='platform', help=f'{platformString}') - parser.add_argument('-id', type=int, required=False, help='bungie ID') + parser.add_argument('--platform', '-p', type=int, required=False, help=f'{platformString}') + parser.add_argument('--membership_id', '-id', type=int, required=False, help='Bungie ID') args = vars(parser.parse_args()) platform = args['platform'] - id = args['id'] + id = args['membership_id'] if platform != None and id != None: USED_MEMBERSHIP = (platform, id) else: MIJAGO = (3, 4611686018482684809) - SUPERQ = (3, 4611686018472661350) - SHTGUNWEDDING = (2, 4611686018428655241) - EURO = (3, 4611686018471254627) - DREDGENQ = (3, 4611686018534347056) - SPRQMAN = (2, 4611686018436271063) - USED_MEMBERSHIP = DREDGENQ + # You can easily set your own ID here: + MYCOOLID = (3, 1234567890123456789) + USED_MEMBERSHIP = MIJAGO - from pathos.multiprocessing import ProcessPool + from pathos.multiprocessing import ProcessPool, ThreadPool, ThreadingPool pathos.helpers.freeze_support() # required for windows pool = ProcessPool() - # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process but takes serious computation power. - # pool = ProcessPool(60) + # You could also specify the amount of threads. Note that this DRASTICALLY speeds up the process but takes serious computation power. + # pool = ProcessPool(40) # check manifest manifest = DestinyManifest().update() - api = BungieApi(os.getenv('BUNGIE_API_KEY')) + # You can also set an api key manually, if you do not want to use environment variables. + API_KEY = os.getenv('BUNGIE_API_KEY') + # API_KEY = "123456789" + + api = BungieApi(API_KEY) # "gif by default, "mp4" if you installed ffmpeg which you should; see README.d VIDEO_TYPE = "mp4" @@ -73,7 +73,7 @@ Director.ClearResultDirectory(*USED_MEMBERSHIP) Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) pc = PGCRCollector(*USED_MEMBERSHIP, api, pool) - pc.getCharacters().getActivities(limit=None).getPGCRs(pagesize=1000) # .combineAllPgcrs() + pc.getCharacters().getActivities(limit=None).getPGCRs() # .combineAllPgcrs() data = pc.getAllPgcrs() pool.close() @@ -101,6 +101,3 @@ Zipper.zip_directory(Director.GetResultDirectory(*USED_MEMBERSHIP), Director.GetZipPath(*USED_MEMBERSHIP)) print("Generated ZIP:", Director.GetZipPath(*USED_MEMBERSHIP)) - - # DiscordSender.send(Director.GetZipPath(*USED_MEMBERSHIP), *USED_MEMBERSHIP) - # print("Sent ZIP:", Director.GetZipPath(*USED_MEMBERSHIP)) From aa0f8a5ec4e8f34d3d2cb2708177c7fa689b29b0 Mon Sep 17 00:00:00 2001 From: Kurt Date: Sun, 21 Apr 2024 20:19:47 -0700 Subject: [PATCH 11/15] Get B.net profile Call bungie api to get profile information. Include display name and display name code to collector. --- app/PgcrCollector.py | 13 +++++++++++++ app/bungieapi.py | 2 +- main.py | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index 842013b..d796eca 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -17,6 +17,19 @@ def __init__(self, membershipType, membershipId, api: BungieApi, pool) -> None: self.api = api self.characters = None self.activities = None + self.displayName = None + + def getProfile(self): + print("> Get profile") + account_profile = self.api.getProfile(self.membershipType, self.membershipId) + bungieGlobalDisplayName = account_profile['profile']['data']['userInfo']['bungieGlobalDisplayName'] + bungieGlobalDisplayNameCode = account_profile['profile']['data']['userInfo']['bungieGlobalDisplayNameCode'] + self.displayName = f'{bungieGlobalDisplayName}[{bungieGlobalDisplayNameCode}]' + print(f"> Found profile: {self.displayName}") + return self + + def getDisplayName(self): + return self.displayName def getCharacters(self): print("> Get Characters") diff --git a/app/bungieapi.py b/app/bungieapi.py index 0362f0f..dbff8ef 100644 --- a/app/bungieapi.py +++ b/app/bungieapi.py @@ -11,7 +11,7 @@ def __init__(self, api_key: str): self.__HEADERS = {"X-API-Key": api_key} pass - def getProfile(self, membershipType, destinyMembershipId, components=[200]): + def getProfile(self, membershipType, destinyMembershipId, components=[100]): params = {} if components is not None: params["components"] = components diff --git a/main.py b/main.py index 5a901fa..e04c87b 100644 --- a/main.py +++ b/main.py @@ -72,7 +72,9 @@ Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) Director.ClearResultDirectory(*USED_MEMBERSHIP) Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) + pc = PGCRCollector(*USED_MEMBERSHIP, api, pool) + pc.getProfile() pc.getCharacters().getActivities(limit=None).getPGCRs() # .combineAllPgcrs() data = pc.getAllPgcrs() From fc471926f648f7ebf7d0edfcc7a1973b19c51483 Mon Sep 17 00:00:00 2001 From: Kurt Date: Tue, 23 Apr 2024 01:12:25 -0700 Subject: [PATCH 12/15] Collect display name. Store data as display name. Report character classes. Data storage relies on display name instead of platform/id. Makes it easier when running reports on multiple players. Logging shows character IDs, which are deleted, and the class of active characters. Cleaned up activities to more closely match API definitions. Class hashes are stored locally since they are not expected to change. --- app/Director.py | 28 ++++++------- app/PgcrCollector.py | 20 ++++++--- app/bungieapi.py | 17 +++++++- app/bungiemanifest.py | 18 ++++++-- app/data/activities.py | 41 ++++++++++++++---- app/data/classhash.py | 7 ++++ app/reports/ActivityCountReport.py | 4 +- app/reports/ActivityLocationTimeReport.py | 4 +- app/reports/ActivityLocationWeaponReport.py | 4 +- app/reports/ActivityTypeRaceReport.py | 6 +-- app/reports/ActivityWinrateReport.py | 4 +- app/reports/FireteamActivityReport.py | 6 +-- app/reports/FireteamRace.py | 6 +-- app/reports/KDReport.py | 4 +- app/reports/KillsDeathsAssistsReport.py | 4 +- app/reports/LightLevelReport.py | 4 +- app/reports/PlaytimeCharacterReport.py | 4 +- app/reports/PlaytimeReport.py | 4 +- app/reports/ReportBase.py | 5 ++- app/reports/WeaponKillTreeReport.py | 4 +- app/reports/WeaponRaceReport.py | 6 +-- app/reports/WeaponReport.py | 8 ++-- app/reports/WeekdayReport.py | 4 +- main.py | 46 ++++++++++----------- 24 files changed, 162 insertions(+), 96 deletions(-) create mode 100644 app/data/classhash.py diff --git a/app/Director.py b/app/Director.py index fcca756..fcb49dc 100644 --- a/app/Director.py +++ b/app/Director.py @@ -5,27 +5,27 @@ class Director: @staticmethod - def ClearResultDirectory(membershipType, membershipId): - path = Director.GetResultDirectory(membershipType, membershipId) + def ClearResultDirectory(displayName): + path = Director.GetResultDirectory(displayName) shutil.rmtree(path) @staticmethod - def GetZipPath(membershipType, membershipId): - return "./data/%d_%d/charts_%d_%d.zip" % (membershipType, membershipId, membershipType, membershipId) + def GetZipPath(displayName): + return f"./data/{displayName}/charts_{displayName}.zip" @staticmethod - def GetResultDirectory(membershipType, membershipId): - return "./data/%d_%d/result/" % (membershipType, membershipId) + def GetResultDirectory(displayName): + return f"./data/{displayName}/result/" @staticmethod - def GetPGCRDirectory(membershipType, membershipId): - return "./data/%d_%d/pgcr/" % (membershipType, membershipId) + def GetPGCRDirectory(displayName): + return f"./data/{displayName}/pgcr/" @staticmethod - def GetAllPgcrFilename(membershipType, membershipId): - return "./data/%d_%d/pgcr.json" % (membershipType, membershipId) - + def GetAllPgcrFilename(displayName): + return f"./data/{displayName}/pgcr.json" + @staticmethod - def CreateDirectoriesForUser(membershipType, membershipId): - Path(Director.GetResultDirectory(membershipType, membershipId)).mkdir(parents=True, exist_ok=True) - Path(Director.GetPGCRDirectory(membershipType, membershipId)).mkdir(parents=True, exist_ok=True) + def CreateDirectoriesForUser(displayName): + Path(Director.GetResultDirectory(displayName)).mkdir(parents=True, exist_ok=True) + Path(Director.GetPGCRDirectory(displayName)).mkdir(parents=True, exist_ok=True) diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index d796eca..54c52c8 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -34,8 +34,16 @@ def getDisplayName(self): def getCharacters(self): print("> Get Characters") account_stats = self.api.getAccountStats(self.membershipType, self.membershipId) - self.characters = [c["characterId"] for c in account_stats["characters"]] - print("Found characters: ", len(self.characters)) + allCharacters = account_stats['characters'] + self.characters = [c["characterId"] for c in allCharacters] + print("> Found characters: ", len(self.characters)) + for char in allCharacters: + deleted = char['deleted'] + if deleted: + className = None + else: + className = self.api.getCharacterClass(self.membershipType, self.membershipId, char['characterId']) + print(f"{char['characterId']}{'' if className == None else ' | ' + className}") return self def getActivities(self, limit=None): @@ -43,7 +51,7 @@ def getActivities(self, limit=None): assert self.characters is not None assert len(self.characters) > 0 - existingPgcrList = [f[5:-5] for f in os.listdir(Director.GetPGCRDirectory(self.membershipType, self.membershipId))] + existingPgcrList = [f[5:-5] for f in os.listdir(Director.GetPGCRDirectory(self.displayName))] self.activities = [] for k, char_id in enumerate(self.characters): @@ -91,7 +99,7 @@ def downloadPGCR(activity): tries += 1 pgcr = bungo.getPGCR(id) - with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.membershipType, self.membershipId), pgcr["activityDetails"]["instanceId"]), "w", encoding='utf-8') as f: + with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.displayName), pgcr["activityDetails"]["instanceId"]), "w", encoding='utf-8') as f: f.write(json.dumps(pgcr)) if len(self.activities) == 0: @@ -105,7 +113,7 @@ def downloadPGCR(activity): def combineAllPgcrs(self): all = self.getAllPgcrs() with Timer("Write all PGCRs to one file"): - with open(Director.GetAllPgcrFilename(self.membershipType, self.membershipId), "w", encoding='utf-8') as f: + with open(Director.GetAllPgcrFilename(self.displayName), "w", encoding='utf-8') as f: json.dump(all, f, ensure_ascii=False) return self @@ -124,7 +132,7 @@ def loadJson(fnameList): return r with Timer("Get all PGCRs from individual files"): - root = Director.GetPGCRDirectory(self.membershipType, self.membershipId) + root = Director.GetPGCRDirectory(self.displayName) fileList = ["%s/%s" % (root, f) for f in os.listdir(root)] chunks = list(zip_longest(*[iter(fileList)] * 100, fillvalue=None)) pgcrs = self.processPool.amap(loadJson, chunks).get() diff --git a/app/bungieapi.py b/app/bungieapi.py index dbff8ef..d6e48b8 100644 --- a/app/bungieapi.py +++ b/app/bungieapi.py @@ -1,5 +1,7 @@ from typing import Dict import requests +from app.data.classhash import CLASS_HASH + API_ROOT_PATH = "https://www.bungie.net/Platform" @@ -48,4 +50,17 @@ def getPGCR(self, activityId): return (api_call.json())['Response'] def getItem(self, itemReferenceId): - pass \ No newline at end of file + pass + + def getCharacterClass(self, membershipType, destinyMembershipId, characterId): + params = {} + params['components'] = 200 + + try: + api_call = requests.get(f'{API_ROOT_PATH}/Destiny2/{membershipType}/Profile/{destinyMembershipId}/Character/{characterId}', headers=self.__HEADERS, params=params, timeout=(10, 10)) + except: + return None + + classHash = (api_call.json())['Response']['character']['data']['classHash'] + + return CLASS_HASH[classHash] \ No newline at end of file diff --git a/app/bungiemanifest.py b/app/bungiemanifest.py index c02c285..91e1652 100644 --- a/app/bungiemanifest.py +++ b/app/bungiemanifest.py @@ -1,6 +1,7 @@ import json, urllib.request, os, shutil from app.data.activities import ACTIVITY_NAMES +from app.data.classhash import CLASS_HASH BUNGIE_BASE = "https://bungie.net/" BUNGIE_API_BASE = "https://bungie.net/Platform/" @@ -18,9 +19,11 @@ def update(self): self.VersionNumber = GetVersionNumber() self.ItemDefinitions = GetInventoryItemDefinitions() self.ActivityNames = GetActivityNames() + self.ClassHash = GetClassDefinition() return self + def GetCacheFolder(): # get current file path and go up two directories path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -28,6 +31,7 @@ def GetCacheFolder(): os.makedirs(path, exist_ok=True) return path + def GetVersionNumber(): print("Check version number") # get version from manifest file @@ -52,12 +56,14 @@ def GetVersionNumber(): with open(file=versionFilePath, mode='w', encoding='utf-8') as f: f.write(version) + def SaveToCache(aDefinition, aJsonBlob): # set root path filePath = os.path.join(GetCacheFolder(), aDefinition) with open(file=filePath, mode='w', encoding='utf-8') as f: json.dump(aJsonBlob, f, ensure_ascii=False, indent=4) + def LoadFromCache(aDefinition): filePath = os.path.join(GetCacheFolder(), aDefinition) exists = os.path.exists(filePath) @@ -67,6 +73,7 @@ def LoadFromCache(aDefinition): blob = json.load(f) return blob, True + def GetManifestDefinitions(definition): print("Get %s" % definition) print("Check %s saved in cache" % definition) @@ -78,20 +85,23 @@ def GetManifestDefinitions(definition): manifestPaths = json.loads(urllib.request.urlopen(BUNGIE_API_BASE + "/Destiny2/Manifest/").read())["Response"] manifestPath = manifestPaths["jsonWorldComponentContentPaths"]["en"][definition] print("Get %s from '%s'" % (definition, BUNGIE_BASE + manifestPath)) - InventoryItemDefinitions = urllib.request.urlopen(BUNGIE_BASE + manifestPath).read() + DefinitionQuery = urllib.request.urlopen(BUNGIE_BASE + manifestPath).read() print("Unpack and parse %s" % definition) - InventoryItemDefinitions = json.loads(InventoryItemDefinitions) + DefinitionQuery = json.loads(DefinitionQuery) print("Json'd %s and cache'd" % definition) - SaveToCache(definition, InventoryItemDefinitions) + SaveToCache(definition, DefinitionQuery) - return InventoryItemDefinitions + return DefinitionQuery def GetInventoryItemDefinitions(): return GetManifestDefinitions("DestinyInventoryItemDefinition") +def GetClassDefinition(): + return CLASS_HASH + def GetActivityNames(): data = GetManifestDefinitions("DestinyActivityDefinition") diff --git a/app/data/activities.py b/app/data/activities.py index 690f893..eb00b23 100644 --- a/app/data/activities.py +++ b/app/data/activities.py @@ -1,17 +1,38 @@ +# https://bungie-net.github.io/multi/schema_Destiny-Entities-Characters-DestinyCharacterActivitiesComponent.html#schema_Destiny-Entities-Characters-DestinyCharacterActivitiesComponent + ACTIVITY_NAMES = { - 0: 'Unknown (0)', - 32: 'Private Matches (all)', + 0: 'None', 2: 'Story', 3: 'Normal Strikes', 4: 'Raid', - 6: 'Explore', + 5: 'All PvP', + 6: 'Patrol', + 7: 'All PvE', + 9: 'Reserved9', + 10: 'Control', + 11: 'Reserved11', + 12: 'Clash', + 13: 'Reserved13', 15: 'Crimson Doubles', 16: 'Nightfall Strikes', 17: 'Prestige Nightfall', + 18: 'All Strikes', + 19: 'Iron Banner', + 20: 'Reserved20', + 21: 'Reserved21', + 22: 'Reserved22', + 24: 'Reserved24', 25: 'Mayhem', + 26: 'Reserved26', + 27: 'Reserved27', + 28: 'Reserved28', + 29: 'Reserved29', + 30: 'Reserved30', 31: 'Supremacy', + 32: 'Private Matches (all)', 37: 'Survival', 38: 'Countdown', + 39: 'Trials of the Nine', 40: 'Social', 41: 'Trials of the Nine Countdown', 42: 'Trials of the Nine Survival', @@ -21,6 +42,7 @@ 46: 'Scored Nightfall Strikes', 47: 'Scored Prestige Nightfall', 48: 'Rumble', + 49: 'Doubles (all)', 50: 'Doubles', 51: 'Private Matches Clash', 52: 'Private Matches Control', @@ -35,10 +57,13 @@ 61: 'Scorched', 62: 'Team Scorched', 63: 'Gambit', + 64: 'PvE Competitive (all)', 65: 'Breakthrough', 66: 'Forge', 67: 'Salvage', 68: 'Iron Banner Salvage', + 69: 'Competitive', + 70: 'Quickplay', 71: 'Clash: Quickplay', 72: 'Clash: Competitive', 73: 'Control: Quickplay', @@ -56,11 +81,11 @@ 85: 'Dares of Eternity', 86: 'Offensive', 87: 'Lost Sector', - 88: "Rift", - 89: "Zone Control", - 90: "Iron Banner Rift", - 91: "Iron Banner Zone Control", - 92: "Relic" + 88: 'Rift', + 89: 'Zone Control', + 90: 'Iron Banner Rift', + 91: 'Iron Banner Zone Control', + 92: 'Relic' } PVP_ACTIVITIES = [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] diff --git a/app/data/classhash.py b/app/data/classhash.py new file mode 100644 index 0000000..f73dfd2 --- /dev/null +++ b/app/data/classhash.py @@ -0,0 +1,7 @@ +# DestinyClassDefinition + +CLASS_HASH = { + 3655393761: 'Titan', + 671679327: 'Hunter', + 2271682572: 'Warlock' +} diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index 0d90417..15623b8 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -10,8 +10,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - activity count" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index 6c74fd4..fb18555 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - activity playtime; by type and location" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index 5ee7f35..070bfd9 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -11,8 +11,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - weapons per activity type and location" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) diff --git a/app/reports/ActivityTypeRaceReport.py b/app/reports/ActivityTypeRaceReport.py index 4377aca..aba217e 100644 --- a/app/reports/ActivityTypeRaceReport.py +++ b/app/reports/ActivityTypeRaceReport.py @@ -12,8 +12,8 @@ def save(self): def getName(self) -> str: return "[ALL] race - activity type playtime" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def save(self): @@ -23,7 +23,7 @@ def generate(self, data): df = self.generateData(data) bcr.bar_chart_race( df=df, - filename=Director.GetResultDirectory(self.membershipType, self.membershipId) + "/" + filename=Director.GetResultDirectory(self.displayName) + "/" + "[ALL] race - activity type playtime." + self.video_type, orientation='h', sort='desc', diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index 4092c36..c1aa23b 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -11,8 +11,8 @@ def save(self): def getName(self) -> str: return "[PVP] chart_tree - activity winrate; per mode and map" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def generate(self, data) -> Report: diff --git a/app/reports/FireteamActivityReport.py b/app/reports/FireteamActivityReport.py index 44fd208..7dd716b 100644 --- a/app/reports/FireteamActivityReport.py +++ b/app/reports/FireteamActivityReport.py @@ -7,14 +7,14 @@ class FireteamActivityReport(Report): def save(self): - with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - fireteam member activities"), "w", encoding='utf-8') as f: + with open("%s/%s.csv" % (Director.GetResultDirectory(self.displayName), "[ALL] table - fireteam member activities"), "w", encoding='utf-8') as f: self.df.to_csv(f, index=False) def getName(self) -> str: return "[ALL] table - fireteam member activities" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.df = None def generate(self, data) -> Report: diff --git a/app/reports/FireteamRace.py b/app/reports/FireteamRace.py index c3e6462..c8f1d05 100644 --- a/app/reports/FireteamRace.py +++ b/app/reports/FireteamRace.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] race - fireteam playtime" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def save(self): @@ -25,7 +25,7 @@ def generate(self, data) -> Report: bcr.bar_chart_race( df=df, - filename=Director.GetResultDirectory(self.membershipType, self.membershipId) + "/" + self.getName() + "." + self.video_type, + filename=Director.GetResultDirectory(self.displayName, ) + "/" + self.getName() + "." + self.video_type, orientation='h', sort='desc', n_bars=20, diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index 0cf7545..c219a25 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -14,8 +14,8 @@ def save(self): def getName(self) -> str: return "[PVP] chart_line - kd and kda ratio" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index 142e207..5316c53 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -14,8 +14,8 @@ def save(self): def getName(self) -> str: return "[PVP] chart_bar - kills, deaths and assists per week" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) diff --git a/app/reports/LightLevelReport.py b/app/reports/LightLevelReport.py index 1538d07..42a9562 100644 --- a/app/reports/LightLevelReport.py +++ b/app/reports/LightLevelReport.py @@ -16,8 +16,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_line - lightlevel" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) diff --git a/app/reports/PlaytimeCharacterReport.py b/app/reports/PlaytimeCharacterReport.py index 0f5091d..db10307 100644 --- a/app/reports/PlaytimeCharacterReport.py +++ b/app/reports/PlaytimeCharacterReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_area - playtime, split per character" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) diff --git a/app/reports/PlaytimeReport.py b/app/reports/PlaytimeReport.py index cccd29b..fd29ba1 100644 --- a/app/reports/PlaytimeReport.py +++ b/app/reports/PlaytimeReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_bar - playtime, weekly and daily" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) diff --git a/app/reports/ReportBase.py b/app/reports/ReportBase.py index 3f2766e..23c8ed6 100644 --- a/app/reports/ReportBase.py +++ b/app/reports/ReportBase.py @@ -6,10 +6,11 @@ class Report: - def __init__(self, membershipType, membershipId, manifest: DestinyManifest) -> None: + def __init__(self, membershipType, membershipId, displayName, manifest: DestinyManifest) -> None: super().__init__() self.membershipType = membershipType self.membershipId = membershipId + self.displayName = displayName self.manifest = manifest self.fig = None @@ -24,6 +25,6 @@ def getName(self) -> str: def save(self): print("Report> Saving %s" % self.getName()) assert self.fig is not None - filename = '%s/%s.html' % (Director.GetResultDirectory(self.membershipType, self.membershipId), self.getName()) + filename = '%s/%s.html' % (Director.GetResultDirectory(self.displayName), self.getName()) with open(filename, "w") as f: f.write(self.fig.to_html(full_html=False, include_plotlyjs='cdn')) diff --git a/app/reports/WeaponKillTreeReport.py b/app/reports/WeaponKillTreeReport.py index 2dcee35..c0295c6 100644 --- a/app/reports/WeaponKillTreeReport.py +++ b/app/reports/WeaponKillTreeReport.py @@ -10,8 +10,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - weapon kills; by type" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index 87a88b3..bf5fbd8 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] race - weapon usage" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def save(self): @@ -30,7 +30,7 @@ def generateintern(self, data, type): df = self.generateData(data, type) bcr.bar_chart_race( df=df, - filename=Director.GetResultDirectory(self.membershipType, self.membershipId) + "/" + filename=Director.GetResultDirectory(self.displayName) + "/" + "[" + type.upper() + "] race - weapon usage." + self.video_type, orientation='h', sort='desc', diff --git a/app/reports/WeaponReport.py b/app/reports/WeaponReport.py index e33b6c3..e9fea7a 100644 --- a/app/reports/WeaponReport.py +++ b/app/reports/WeaponReport.py @@ -13,12 +13,12 @@ class WeaponReport(Report): def save(self): - with open("%s/%s.html" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - weapons used per activity type"), "w") as f: + with open("%s/%s.html" % (Director.GetResultDirectory(self.displayName), "[ALL] table - weapons used per activity type"), "w") as f: f.write("This table shows every weapon you ever used - in any activity. " "Sort order: Type > Weapon Name > Weapon ID > Kills." "
Generated by Mijago") f.write(build_table(self.df, 'blue_light')) - with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - weapons used per activity type"), "w") as f: + with open("%s/%s.csv" % (Director.GetResultDirectory(self.displayName), "[ALL] table - weapons used per activity type"), "w") as f: self.df.to_csv(f, index=False) super().save() @@ -26,8 +26,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_bar - weapons used over time; per activity type " - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.df = None def generate(self, data) -> Report: diff --git a/app/reports/WeekdayReport.py b/app/reports/WeekdayReport.py index affc78d..b808747 100644 --- a/app/reports/WeekdayReport.py +++ b/app/reports/WeekdayReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_heat - playtime per weekday" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: timeArray = self.generateData(data) diff --git a/main.py b/main.py index e04c87b..d295fdc 100644 --- a/main.py +++ b/main.py @@ -54,7 +54,6 @@ from pathos.multiprocessing import ProcessPool, ThreadPool, ThreadingPool pathos.helpers.freeze_support() # required for windows - pool = ProcessPool() # You could also specify the amount of threads. Note that this DRASTICALLY speeds up the process but takes serious computation power. # pool = ProcessPool(40) @@ -66,37 +65,38 @@ # API_KEY = "123456789" api = BungieApi(API_KEY) - # "gif by default, "mp4" if you installed ffmpeg which you should; see README.d + # "mp4" if you installed ffmpeg which you should; see README.d. otherwise "gif" if you do not. VIDEO_TYPE = "mp4" - Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) - Director.ClearResultDirectory(*USED_MEMBERSHIP) - Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) - pc = PGCRCollector(*USED_MEMBERSHIP, api, pool) - pc.getProfile() + displayName = pc.getProfile().getDisplayName() + + Director.CreateDirectoriesForUser(displayName) + Director.ClearResultDirectory(displayName) + Director.CreateDirectoriesForUser(displayName) + pc.getCharacters().getActivities(limit=None).getPGCRs() # .combineAllPgcrs() data = pc.getAllPgcrs() pool.close() reports = [ - ActivityCountReport(*USED_MEMBERSHIP, manifest), - ActivityLocationTimeReport(*USED_MEMBERSHIP, manifest), - ActivityLocationWeaponReport(*USED_MEMBERSHIP, manifest), - ActivityTypeRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - ActivityWinrateReport(*USED_MEMBERSHIP, manifest), - FireteamActivityReport(*USED_MEMBERSHIP, manifest), - FireteamRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - KDReport(*USED_MEMBERSHIP, manifest), - KillsDeathsAssistsReport(*USED_MEMBERSHIP, manifest), - LightLevelReport(*USED_MEMBERSHIP, manifest), - PlaytimeCharacterReport(*USED_MEMBERSHIP, manifest), - PlaytimeReport(*USED_MEMBERSHIP, manifest), - WeaponKillTreeReport(*USED_MEMBERSHIP, manifest), - WeaponRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - WeaponReport(*USED_MEMBERSHIP, manifest), - WeekdayReport(*USED_MEMBERSHIP, manifest) + ActivityCountReport(*USED_MEMBERSHIP, displayName, manifest), + ActivityLocationTimeReport(*USED_MEMBERSHIP, displayName, manifest), + ActivityLocationWeaponReport(*USED_MEMBERSHIP, displayName, manifest), + ActivityTypeRaceReport(*USED_MEMBERSHIP, displayName, manifest, video_type=VIDEO_TYPE), + ActivityWinrateReport(*USED_MEMBERSHIP, displayName, manifest), + FireteamActivityReport(*USED_MEMBERSHIP, displayName, manifest), + FireteamRaceReport(*USED_MEMBERSHIP, displayName, manifest, video_type=VIDEO_TYPE), + KDReport(*USED_MEMBERSHIP, displayName, manifest), + KillsDeathsAssistsReport(*USED_MEMBERSHIP, displayName, manifest), + LightLevelReport(*USED_MEMBERSHIP, displayName, manifest), + PlaytimeCharacterReport(*USED_MEMBERSHIP, displayName, manifest), + PlaytimeReport(*USED_MEMBERSHIP, displayName, manifest), + WeaponKillTreeReport(*USED_MEMBERSHIP, displayName, manifest), + WeaponRaceReport(*USED_MEMBERSHIP, displayName, manifest, video_type=VIDEO_TYPE), + WeaponReport(*USED_MEMBERSHIP, displayName, manifest), + WeekdayReport(*USED_MEMBERSHIP, displayName, manifest) ] for report in reports: report.generate(data).save() From d667f2216b4279fbc724dc2257074da7d3416df2 Mon Sep 17 00:00:00 2001 From: Kurt Date: Tue, 23 Apr 2024 09:28:09 -0700 Subject: [PATCH 13/15] Put back process pool. Ignore Mac remant. --- .gitignore | 3 +++ main.py | 1 + 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 9ba3c69..c4832b6 100644 --- a/.gitignore +++ b/.gitignore @@ -181,3 +181,6 @@ pyrightconfig.json # visual studio remnants /.vscode + +# mac remnants +.DS_Store diff --git a/main.py b/main.py index d295fdc..5799a40 100644 --- a/main.py +++ b/main.py @@ -54,6 +54,7 @@ from pathos.multiprocessing import ProcessPool, ThreadPool, ThreadingPool pathos.helpers.freeze_support() # required for windows + pool = ProcessPool() # You could also specify the amount of threads. Note that this DRASTICALLY speeds up the process but takes serious computation power. # pool = ProcessPool(40) From f44def3af7e233176415b70babf95eb33500cef7 Mon Sep 17 00:00:00 2001 From: Kurt Date: Fri, 26 Apr 2024 10:18:40 -0700 Subject: [PATCH 14/15] Cleanup reporting logging. Fix zipper paths to use display name. --- app/PgcrCollector.py | 4 ++-- app/reports/ActivityCountReport.py | 2 +- app/reports/ActivityLocationTimeReport.py | 2 +- app/reports/ActivityLocationWeaponReport.py | 2 +- app/reports/ActivityTypeRaceReport.py | 2 +- app/reports/ActivityWinrateReport.py | 4 +--- app/reports/FireteamActivityReport.py | 2 +- app/reports/FireteamRace.py | 2 +- app/reports/KDReport.py | 2 +- app/reports/KillsDeathsAssistsReport.py | 2 +- app/reports/LightLevelReport.py | 2 +- app/reports/PlaytimeCharacterReport.py | 2 +- app/reports/PlaytimeReport.py | 2 +- app/reports/WeaponKillTreeReport.py | 2 +- app/reports/WeaponRaceReport.py | 2 +- app/reports/WeaponReport.py | 2 +- app/reports/WeekdayReport.py | 2 +- main.py | 4 ++-- 18 files changed, 20 insertions(+), 22 deletions(-) diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index 54c52c8..814054f 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -25,7 +25,7 @@ def getProfile(self): bungieGlobalDisplayName = account_profile['profile']['data']['userInfo']['bungieGlobalDisplayName'] bungieGlobalDisplayNameCode = account_profile['profile']['data']['userInfo']['bungieGlobalDisplayNameCode'] self.displayName = f'{bungieGlobalDisplayName}[{bungieGlobalDisplayNameCode}]' - print(f"> Found profile: {self.displayName}") + print(f"Found profile: {self.displayName}") return self def getDisplayName(self): @@ -36,7 +36,7 @@ def getCharacters(self): account_stats = self.api.getAccountStats(self.membershipType, self.membershipId) allCharacters = account_stats['characters'] self.characters = [c["characterId"] for c in allCharacters] - print("> Found characters: ", len(self.characters)) + print("Found characters: ", len(self.characters)) for char in allCharacters: deleted = char['deleted'] if deleted: diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index 15623b8..50848c7 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -35,7 +35,7 @@ def generateData(self, data): mode = [] directorActivity = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index fb18555..b6a5d23 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -35,7 +35,7 @@ def generateData(self, data): activity = [] directorActivity = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index 070bfd9..4acd429 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -38,7 +38,7 @@ def generateData(self, data): weapon = [] directorActivity = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/ActivityTypeRaceReport.py b/app/reports/ActivityTypeRaceReport.py index aba217e..57d21cf 100644 --- a/app/reports/ActivityTypeRaceReport.py +++ b/app/reports/ActivityTypeRaceReport.py @@ -56,7 +56,7 @@ def generateData(self, datap): # pve, pvp, gambit eps = [] - for data in tqdm(datap): + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue date = parser.parse(data["period"]) # find own user entry diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index c1aa23b..da3d0eb 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -37,7 +37,7 @@ def generateData(self, data): directorActivity = [] wintype = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: @@ -74,8 +74,6 @@ def generateData(self, data): else: directorActivity.append(key) - print("done") - df = pd.DataFrame({ "type": typ, "mode": mode, diff --git a/app/reports/FireteamActivityReport.py b/app/reports/FireteamActivityReport.py index 7dd716b..c30d7e0 100644 --- a/app/reports/FireteamActivityReport.py +++ b/app/reports/FireteamActivityReport.py @@ -28,7 +28,7 @@ def generateListDataframe(self, datap): displayNames = dict() displayNameTimes = dict() - for data in tqdm(datap): + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue # find own user entry entry = [e for e in data["entries"] if e["player"]["destinyUserInfo"]["membershipId"] == str(self.membershipId)][0] diff --git a/app/reports/FireteamRace.py b/app/reports/FireteamRace.py index c8f1d05..90f2265 100644 --- a/app/reports/FireteamRace.py +++ b/app/reports/FireteamRace.py @@ -55,7 +55,7 @@ def generateData(self, datap): displayNames = dict() displayNameTimes = dict() - for data in tqdm(datap): + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue # find own user entry entry = [e for e in data["entries"] if e["player"]["destinyUserInfo"]["membershipId"] == str(self.membershipId)][0] diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index c219a25..45fb082 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -51,7 +51,7 @@ def generateRawDataframe(self, data): deaths = [] kills = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index 5316c53..f25a879 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -51,7 +51,7 @@ def generateRawDataframe(self, data): deaths = [] kills = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/LightLevelReport.py b/app/reports/LightLevelReport.py index 42a9562..bed6a61 100644 --- a/app/reports/LightLevelReport.py +++ b/app/reports/LightLevelReport.py @@ -42,7 +42,7 @@ def generateRawDataframe(self, data): endtime = [] lightlevel = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/PlaytimeCharacterReport.py b/app/reports/PlaytimeCharacterReport.py index db10307..b9688bb 100644 --- a/app/reports/PlaytimeCharacterReport.py +++ b/app/reports/PlaytimeCharacterReport.py @@ -48,7 +48,7 @@ def generateRawDataframe(self, data): clazz = [] character = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/PlaytimeReport.py b/app/reports/PlaytimeReport.py index fd29ba1..5d40559 100644 --- a/app/reports/PlaytimeReport.py +++ b/app/reports/PlaytimeReport.py @@ -42,7 +42,7 @@ def generateRawDataframe(self, data): playtime = [] mode = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/WeaponKillTreeReport.py b/app/reports/WeaponKillTreeReport.py index c0295c6..19cccda 100644 --- a/app/reports/WeaponKillTreeReport.py +++ b/app/reports/WeaponKillTreeReport.py @@ -34,7 +34,7 @@ def generateData(self, data): killType = [] kills = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index bf5fbd8..ccef723 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -60,7 +60,7 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit eps = [] - for data in tqdm(datap): + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue date = parser.parse(data["period"]) # find own user entry diff --git a/app/reports/WeaponReport.py b/app/reports/WeaponReport.py index e9fea7a..3d8e098 100644 --- a/app/reports/WeaponReport.py +++ b/app/reports/WeaponReport.py @@ -64,7 +64,7 @@ def generateRawDataframe(self, data): mode_name = [] kills_precision = [] - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/WeekdayReport.py b/app/reports/WeekdayReport.py index b808747..286beb5 100644 --- a/app/reports/WeekdayReport.py +++ b/app/reports/WeekdayReport.py @@ -56,7 +56,7 @@ def generateKey(xdate): xdate.hour ) - for datapoint in tqdm(data): + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/main.py b/main.py index 5799a40..f2348b6 100644 --- a/main.py +++ b/main.py @@ -102,5 +102,5 @@ for report in reports: report.generate(data).save() - Zipper.zip_directory(Director.GetResultDirectory(*USED_MEMBERSHIP), Director.GetZipPath(*USED_MEMBERSHIP)) - print("Generated ZIP:", Director.GetZipPath(*USED_MEMBERSHIP)) + Zipper.zip_directory(Director.GetResultDirectory(displayName), Director.GetZipPath(displayName)) + print("Generated ZIP:", Director.GetZipPath(displayName)) From baadd577c40388de2341c10bee19c427b30f4f22 Mon Sep 17 00:00:00 2001 From: Kurt Anderson Date: Fri, 27 Dec 2024 11:14:01 -0800 Subject: [PATCH 15/15] Final 2024 commit --- .gitignore | 3 +++ app/PgcrCollector.py | 2 +- main.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index c4832b6..c83bae2 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,6 @@ pyrightconfig.json # mac remnants .DS_Store + +# build scripts +*.sh diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index 814054f..44a0d65 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -106,7 +106,7 @@ def downloadPGCR(activity): print("No activities to grab") return self - from tqdm.auto import tqdm + from tqdm.auto import tqdm list(tqdm(self.processPool.imap(downloadPGCR, self.activities), total=len(self.activities), desc="Downloading PGCRs")) return self diff --git a/main.py b/main.py index f2348b6..5e5f90d 100644 --- a/main.py +++ b/main.py @@ -30,7 +30,7 @@ # build argument parsing descriptionString = """Get and compile stats for a Destiny 2 user. - example: main.py -p 3 -id 4611686018482684809""" + example: main.py -p 3 -id 4611686018472661350""" platformString = """ Xbox 1 Psn 2 Steam 3 @@ -56,7 +56,7 @@ pathos.helpers.freeze_support() # required for windows pool = ProcessPool() # You could also specify the amount of threads. Note that this DRASTICALLY speeds up the process but takes serious computation power. - # pool = ProcessPool(40) + # pool = ProcessPool(128) # check manifest manifest = DestinyManifest().update()