diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c83bae2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,189 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python + +# ignore generated output data +/data +/cache + +# visual studio remnants +/.vscode + +# mac remnants +.DS_Store + +# build scripts +*.sh diff --git a/README.md b/README.md index ef436f1..f5d1494 100644 --- a/README.md +++ b/README.md @@ -9,22 +9,24 @@ My Twitter: https://twitter.com/MijagoCoding/ # How to Use? 3) Install all required packages - 1) `python -m pip install pandas plotly pathos requests pretty_html_table bar-chart-race` - 2) If you want to use mp4 instead of gif, also install `python -m pip install python-ffmpeg` and put a [ffmpeg](https://www.ffmpeg.org/download.html) in your PATH variable. Then set the `VIDEO_TYPE` in `main.py` to `mp4`. + 1) `python3 -m pip install pandas plotly pathos requests pretty_html_table bar-chart-race tqdm` + 2) If you want to use mp4 instead of gif, also install `python3 -m pip install python-ffmpeg` and put a [ffmpeg](https://www.ffmpeg.org/download.html) in your PATH variable. Then set the `VIDEO_TYPE` in `main.py` to `mp4`. **I highly encourage you to do this, as the gifs tend to be 40mb in size, whereas the mp4 is only around 1.5mb~2mb**. Download it here: [ffmpeg](https://github.com/BtbN/FFmpeg-Builds/releases) (for Windows, `ffmpeg-n5.0-latest-win64-gpl-5.0.zip`). -4) Add your api key to `main.py`. For this, edit `api = BungieApi("API-KEY")`. Get it [here](https://www.bungie.net/en/Application). -5) Edit your user info in `main.py`. +4) Set your API key as an environemnt variable `BUNGIE_API_KEY`. Get the key [here](https://www.bungie.net/en/Application). + 1) Alternatively: Add your api key to `main.py`. For this, edit `# API_KEY = "123456789"`. +5) Edit your user info in `main.py`. Alternatively, you can also use command line parameters to set this later. ```py MEMBERSHIP_MIJAGO = (3, 4611686018482684809) MEMBERSHIP_MYCOOLID = (1, 1231231231231233353) # for example, add this USED_MEMBERSHIP = MEMBERSHIP_MYCOOLID - api = BungieApi("API-KEY") ``` -6) Run! `python3 main.py` - 1) May take a while. I need 35~45 seconds for 1000 PGCRs with a download speed of 4.5mb/s. +6) Run the script `python3 main.py`. + 1) Complete Example: `BUNGIE_API_KEY=123456789012345 python3 main.py -p 3 -id 4611686018482684809` + 2) Alternatively you can also specify the platform and user: `python3 main.py -p 3 -id 4611686018482684809` + 3) This may take a while. I need 35~45 seconds for 1000 PGCRs with a download speed of 4.5mb/s. # Where do I get my user ID? 1) Go to https://www.d2checklist.com (or any other similar page) diff --git a/app/Director.py b/app/Director.py index fcca756..fcb49dc 100644 --- a/app/Director.py +++ b/app/Director.py @@ -5,27 +5,27 @@ class Director: @staticmethod - def ClearResultDirectory(membershipType, membershipId): - path = Director.GetResultDirectory(membershipType, membershipId) + def ClearResultDirectory(displayName): + path = Director.GetResultDirectory(displayName) shutil.rmtree(path) @staticmethod - def GetZipPath(membershipType, membershipId): - return "./data/%d_%d/charts_%d_%d.zip" % (membershipType, membershipId, membershipType, membershipId) + def GetZipPath(displayName): + return f"./data/{displayName}/charts_{displayName}.zip" @staticmethod - def GetResultDirectory(membershipType, membershipId): - return "./data/%d_%d/result/" % (membershipType, membershipId) + def GetResultDirectory(displayName): + return f"./data/{displayName}/result/" @staticmethod - def GetPGCRDirectory(membershipType, membershipId): - return "./data/%d_%d/pgcr/" % (membershipType, membershipId) + def GetPGCRDirectory(displayName): + return f"./data/{displayName}/pgcr/" @staticmethod - def GetAllPgcrFilename(membershipType, membershipId): - return "./data/%d_%d/pgcr.json" % (membershipType, membershipId) - + def GetAllPgcrFilename(displayName): + return f"./data/{displayName}/pgcr.json" + @staticmethod - def CreateDirectoriesForUser(membershipType, membershipId): - Path(Director.GetResultDirectory(membershipType, membershipId)).mkdir(parents=True, exist_ok=True) - Path(Director.GetPGCRDirectory(membershipType, membershipId)).mkdir(parents=True, exist_ok=True) + def CreateDirectoriesForUser(displayName): + Path(Director.GetResultDirectory(displayName)).mkdir(parents=True, exist_ok=True) + Path(Director.GetPGCRDirectory(displayName)).mkdir(parents=True, exist_ok=True) diff --git a/app/DiscordSender.py b/app/DiscordSender.py deleted file mode 100644 index 47f05fb..0000000 --- a/app/DiscordSender.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -import discord - -from app.Director import Director - - -class DiscordSender: - @staticmethod - def send(zipPath, membershipType, membershipId): - # DISCORD WEBHOOK - webhook = discord.Webhook.partial(935137211285802, 'q64vyCdgFKCdq5pdhfoU95UEKD7vwolUSAb7SKFJyX5iP3pHWJ5G964fp7s3xDlRb', adapter=discord.RequestsWebhookAdapter()) # Your webhook - - with open(zipPath, "rb") as f: - zipFile = discord.File(f, filename="Report_%d_%d.zip" % (membershipType, membershipId)) - - webhook.send("Report for %d_%d" % (membershipType, membershipId), username="Mijago's PgcrReport Generator", file=zipFile) diff --git a/app/PgcrCollector.py b/app/PgcrCollector.py index 01c9adb..44a0d65 100644 --- a/app/PgcrCollector.py +++ b/app/PgcrCollector.py @@ -17,12 +17,33 @@ def __init__(self, membershipType, membershipId, api: BungieApi, pool) -> None: self.api = api self.characters = None self.activities = None + self.displayName = None + + def getProfile(self): + print("> Get profile") + account_profile = self.api.getProfile(self.membershipType, self.membershipId) + bungieGlobalDisplayName = account_profile['profile']['data']['userInfo']['bungieGlobalDisplayName'] + bungieGlobalDisplayNameCode = account_profile['profile']['data']['userInfo']['bungieGlobalDisplayNameCode'] + self.displayName = f'{bungieGlobalDisplayName}[{bungieGlobalDisplayNameCode}]' + print(f"Found profile: {self.displayName}") + return self + + def getDisplayName(self): + return self.displayName def getCharacters(self): print("> Get Characters") account_stats = self.api.getAccountStats(self.membershipType, self.membershipId) - self.characters = [c["characterId"] for c in account_stats["characters"]] + allCharacters = account_stats['characters'] + self.characters = [c["characterId"] for c in allCharacters] print("Found characters: ", len(self.characters)) + for char in allCharacters: + deleted = char['deleted'] + if deleted: + className = None + else: + className = self.api.getCharacterClass(self.membershipType, self.membershipId, char['characterId']) + print(f"{char['characterId']}{'' if className == None else ' | ' + className}") return self def getActivities(self, limit=None): @@ -30,7 +51,7 @@ def getActivities(self, limit=None): assert self.characters is not None assert len(self.characters) > 0 - existingPgcrList = [f[5:-5] for f in os.listdir(Director.GetPGCRDirectory(self.membershipType, self.membershipId))] + existingPgcrList = [f[5:-5] for f in os.listdir(Director.GetPGCRDirectory(self.displayName))] self.activities = [] for k, char_id in enumerate(self.characters): @@ -66,7 +87,7 @@ def downloadActivityPage(page): return self - def getPGCRs(self, pagesize=1000): + def getPGCRs(self): bungo = self.api def downloadPGCR(activity): @@ -78,29 +99,21 @@ def downloadPGCR(activity): tries += 1 pgcr = bungo.getPGCR(id) - with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.membershipType, self.membershipId), pgcr["activityDetails"]["instanceId"]), "w") as f: + with open("%s/pgcr_%s.json" % (Director.GetPGCRDirectory(self.displayName), pgcr["activityDetails"]["instanceId"]), "w", encoding='utf-8') as f: f.write(json.dumps(pgcr)) - stepsize = pagesize - START_PAGE = 0 - if len(self.activities) == 0: print("No activities to grab") return self - for steps in range(START_PAGE, (len(self.activities) + stepsize - 1) // stepsize): - try: - with Timer("Get PGCRs %d through %d" % (steps * stepsize + 1, min(len(self.activities), (steps + 1) * stepsize))): - # self.processPool.restart(True) - self.processPool.amap(downloadPGCR, self.activities[steps * stepsize:(steps + 1) * stepsize]).get() - except Exception as e: - print(e) + from tqdm.auto import tqdm + list(tqdm(self.processPool.imap(downloadPGCR, self.activities), total=len(self.activities), desc="Downloading PGCRs")) return self def combineAllPgcrs(self): all = self.getAllPgcrs() with Timer("Write all PGCRs to one file"): - with open(Director.GetAllPgcrFilename(self.membershipType, self.membershipId), "w", encoding='utf-8') as f: + with open(Director.GetAllPgcrFilename(self.displayName), "w", encoding='utf-8') as f: json.dump(all, f, ensure_ascii=False) return self @@ -111,12 +124,15 @@ def loadJson(fnameList): for fname in fnameList: if fname is None: continue - with open(fname, "r") as f: - r.append( json.load(f)) + with open(fname, "r", encoding='utf-8') as f: + try: + r.append(json.load(f)) + except Exception: + print('Error on %s' % fname) return r with Timer("Get all PGCRs from individual files"): - root = Director.GetPGCRDirectory(self.membershipType, self.membershipId) + root = Director.GetPGCRDirectory(self.displayName) fileList = ["%s/%s" % (root, f) for f in os.listdir(root)] chunks = list(zip_longest(*[iter(fileList)] * 100, fillvalue=None)) pgcrs = self.processPool.amap(loadJson, chunks).get() diff --git a/app/bungieapi.py b/app/bungieapi.py index 0362f0f..d6e48b8 100644 --- a/app/bungieapi.py +++ b/app/bungieapi.py @@ -1,5 +1,7 @@ from typing import Dict import requests +from app.data.classhash import CLASS_HASH + API_ROOT_PATH = "https://www.bungie.net/Platform" @@ -11,7 +13,7 @@ def __init__(self, api_key: str): self.__HEADERS = {"X-API-Key": api_key} pass - def getProfile(self, membershipType, destinyMembershipId, components=[200]): + def getProfile(self, membershipType, destinyMembershipId, components=[100]): params = {} if components is not None: params["components"] = components @@ -48,4 +50,17 @@ def getPGCR(self, activityId): return (api_call.json())['Response'] def getItem(self, itemReferenceId): - pass \ No newline at end of file + pass + + def getCharacterClass(self, membershipType, destinyMembershipId, characterId): + params = {} + params['components'] = 200 + + try: + api_call = requests.get(f'{API_ROOT_PATH}/Destiny2/{membershipType}/Profile/{destinyMembershipId}/Character/{characterId}', headers=self.__HEADERS, params=params, timeout=(10, 10)) + except: + return None + + classHash = (api_call.json())['Response']['character']['data']['classHash'] + + return CLASS_HASH[classHash] \ No newline at end of file diff --git a/app/bungiemanifest.py b/app/bungiemanifest.py index 0a68d41..91e1652 100644 --- a/app/bungiemanifest.py +++ b/app/bungiemanifest.py @@ -1,42 +1,107 @@ -import json, urllib.request +import json, urllib.request, os, shutil from app.data.activities import ACTIVITY_NAMES +from app.data.classhash import CLASS_HASH BUNGIE_BASE = "https://bungie.net/" BUNGIE_API_BASE = "https://bungie.net/Platform/" - class DestinyManifest(): def __init__(self): self.ActivityNames = None self.ActivityTypeNames = None self.ItemDefinitions = None + self.CacheFolder = None def update(self): + self.CacheFolder = GetCacheFolder() self.ActivityTypeNames = GetActivityTypeNames() + self.VersionNumber = GetVersionNumber() self.ItemDefinitions = GetInventoryItemDefinitions() self.ActivityNames = GetActivityNames() + self.ClassHash = GetClassDefinition() return self + + +def GetCacheFolder(): + # get current file path and go up two directories + path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + path = os.path.join(path, 'cache') + os.makedirs(path, exist_ok=True) + return path + + +def GetVersionNumber(): + print("Check version number") + # get version from manifest file + manifestPaths = json.loads(urllib.request.urlopen(BUNGIE_API_BASE + "/Destiny2/Manifest/").read())["Response"] + version = manifestPaths['version'] + cacheRoot = GetCacheFolder() + + # if version file exists and matches response, continue + if os.path.exists(os.path.join(cacheRoot, version)): + print("Version check passed") + return + + # if version file does not exist or does not match + print("Version check failed, deleting cache") + # delete existing cache folder + shutil.rmtree(cacheRoot, ignore_errors=True) + print("Updating version") + # create cache folder + GetCacheFolder() + versionFilePath = os.path.join(cacheRoot, version) + # create new version file + with open(file=versionFilePath, mode='w', encoding='utf-8') as f: + f.write(version) + + +def SaveToCache(aDefinition, aJsonBlob): + # set root path + filePath = os.path.join(GetCacheFolder(), aDefinition) + with open(file=filePath, mode='w', encoding='utf-8') as f: + json.dump(aJsonBlob, f, ensure_ascii=False, indent=4) + + +def LoadFromCache(aDefinition): + filePath = os.path.join(GetCacheFolder(), aDefinition) + exists = os.path.exists(filePath) + if not exists: + return None, False + with open(file=filePath, mode='r', encoding='utf-8') as f: + blob = json.load(f) + return blob, True def GetManifestDefinitions(definition): print("Get %s" % definition) + print("Check %s saved in cache" % definition) + blob, exists = LoadFromCache(definition) + if exists: + print("Found %s in cache" % definition) + return blob + manifestPaths = json.loads(urllib.request.urlopen(BUNGIE_API_BASE + "/Destiny2/Manifest/").read())["Response"] manifestPath = manifestPaths["jsonWorldComponentContentPaths"]["en"][definition] print("Get %s from '%s'" % (definition, BUNGIE_BASE + manifestPath)) - InventoryItemDefinitions = urllib.request.urlopen(BUNGIE_BASE + manifestPath).read() + DefinitionQuery = urllib.request.urlopen(BUNGIE_BASE + manifestPath).read() print("Unpack and parse %s" % definition) - InventoryItemDefinitions = json.loads(InventoryItemDefinitions) - print("Json'd %s" % definition) - return InventoryItemDefinitions + DefinitionQuery = json.loads(DefinitionQuery) + print("Json'd %s and cache'd" % definition) + SaveToCache(definition, DefinitionQuery) + + return DefinitionQuery def GetInventoryItemDefinitions(): return GetManifestDefinitions("DestinyInventoryItemDefinition") +def GetClassDefinition(): + return CLASS_HASH + def GetActivityNames(): data = GetManifestDefinitions("DestinyActivityDefinition") diff --git a/app/data/activities.py b/app/data/activities.py index 171b830..eb00b23 100644 --- a/app/data/activities.py +++ b/app/data/activities.py @@ -1,17 +1,38 @@ +# https://bungie-net.github.io/multi/schema_Destiny-Entities-Characters-DestinyCharacterActivitiesComponent.html#schema_Destiny-Entities-Characters-DestinyCharacterActivitiesComponent + ACTIVITY_NAMES = { - 0: 'Unknown (0)', - 32: 'Private Matches (all)', + 0: 'None', 2: 'Story', 3: 'Normal Strikes', 4: 'Raid', - 6: 'Explore', + 5: 'All PvP', + 6: 'Patrol', + 7: 'All PvE', + 9: 'Reserved9', + 10: 'Control', + 11: 'Reserved11', + 12: 'Clash', + 13: 'Reserved13', 15: 'Crimson Doubles', 16: 'Nightfall Strikes', 17: 'Prestige Nightfall', + 18: 'All Strikes', + 19: 'Iron Banner', + 20: 'Reserved20', + 21: 'Reserved21', + 22: 'Reserved22', + 24: 'Reserved24', 25: 'Mayhem', + 26: 'Reserved26', + 27: 'Reserved27', + 28: 'Reserved28', + 29: 'Reserved29', + 30: 'Reserved30', 31: 'Supremacy', + 32: 'Private Matches (all)', 37: 'Survival', 38: 'Countdown', + 39: 'Trials of the Nine', 40: 'Social', 41: 'Trials of the Nine Countdown', 42: 'Trials of the Nine Survival', @@ -21,6 +42,7 @@ 46: 'Scored Nightfall Strikes', 47: 'Scored Prestige Nightfall', 48: 'Rumble', + 49: 'Doubles (all)', 50: 'Doubles', 51: 'Private Matches Clash', 52: 'Private Matches Control', @@ -35,10 +57,13 @@ 61: 'Scorched', 62: 'Team Scorched', 63: 'Gambit', + 64: 'PvE Competitive (all)', 65: 'Breakthrough', 66: 'Forge', 67: 'Salvage', 68: 'Iron Banner Salvage', + 69: 'Competitive', + 70: 'Quickplay', 71: 'Clash: Quickplay', 72: 'Clash: Competitive', 73: 'Control: Quickplay', @@ -56,10 +81,12 @@ 85: 'Dares of Eternity', 86: 'Offensive', 87: 'Lost Sector', - 88: "Rift", - 89: "Zone Control", - 90: "Iron Banner Rift" + 88: 'Rift', + 89: 'Zone Control', + 90: 'Iron Banner Rift', + 91: 'Iron Banner Zone Control', + 92: 'Relic' } -PVP_ACTIVITIES = [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] +PVP_ACTIVITIES = [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15] GAMBIT_ACTIVITIES = [75, 63] diff --git a/app/data/classhash.py b/app/data/classhash.py new file mode 100644 index 0000000..f73dfd2 --- /dev/null +++ b/app/data/classhash.py @@ -0,0 +1,7 @@ +# DestinyClassDefinition + +CLASS_HASH = { + 3655393761: 'Titan', + 671679327: 'Hunter', + 2271682572: 'Warlock' +} diff --git a/app/reports/ActivityCountReport.py b/app/reports/ActivityCountReport.py index b293d41..50848c7 100644 --- a/app/reports/ActivityCountReport.py +++ b/app/reports/ActivityCountReport.py @@ -10,8 +10,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - activity count" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) @@ -29,17 +29,18 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm typ = [] mode = [] directorActivity = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue typus = "PvE" - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61,32, 60, 59, 32, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" @@ -62,4 +63,5 @@ def generateData(self, data): }) df = df.groupby(["type", "mode", "directorActivity"]).size().reset_index(name='count') - return df \ No newline at end of file + return df + \ No newline at end of file diff --git a/app/reports/ActivityLocationTimeReport.py b/app/reports/ActivityLocationTimeReport.py index bc79f03..b6a5d23 100644 --- a/app/reports/ActivityLocationTimeReport.py +++ b/app/reports/ActivityLocationTimeReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - activity playtime; by type and location" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) @@ -29,12 +29,13 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm category = [] playtime = [] activity = [] directorActivity = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: @@ -46,7 +47,7 @@ def generateData(self, data): start_date = datetime.fromtimestamp(timestamp + starts) typus = "PvE" - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityLocationWeaponReport.py b/app/reports/ActivityLocationWeaponReport.py index 4bc5db1..4acd429 100644 --- a/app/reports/ActivityLocationWeaponReport.py +++ b/app/reports/ActivityLocationWeaponReport.py @@ -11,8 +11,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - weapons per activity type and location" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) @@ -30,6 +30,7 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm category = [] kills = [] @@ -37,7 +38,7 @@ def generateData(self, data): weapon = [] directorActivity = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: @@ -47,7 +48,7 @@ def generateData(self, data): for wp in entry["extended"]["weapons"]: typus = "PvE" - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" diff --git a/app/reports/ActivityTypeRaceReport.py b/app/reports/ActivityTypeRaceReport.py index 52d617d..57d21cf 100644 --- a/app/reports/ActivityTypeRaceReport.py +++ b/app/reports/ActivityTypeRaceReport.py @@ -12,8 +12,8 @@ def save(self): def getName(self) -> str: return "[ALL] race - activity type playtime" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def save(self): @@ -23,7 +23,7 @@ def generate(self, data): df = self.generateData(data) bcr.bar_chart_race( df=df, - filename=Director.GetResultDirectory(self.membershipType, self.membershipId) + "/" + filename=Director.GetResultDirectory(self.displayName) + "/" + "[ALL] race - activity type playtime." + self.video_type, orientation='h', sort='desc', @@ -50,9 +50,13 @@ def generate(self, data): return self def generateData(self, datap): # pve, pvp, gambit + from tqdm import tqdm + import warnings + warnings.simplefilter("ignore") + eps = [] - for data in datap: + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue date = parser.parse(data["period"]) # find own user entry @@ -67,8 +71,8 @@ def generateData(self, datap): # pve, pvp, gambit ) df = pd.DataFrame(eps, columns=["date", "name", "hours"]) - df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum().reset_index() - df2["cumsum"] = df2.groupby(["name"]).cumsum() + df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum(numeric_only=True).reset_index() + df2["cumsum"] = df2.groupby(["name"]).cumsum(numeric_only=True) df3 = df2.pivot(index="date", columns="name", values="cumsum") df3 = df3.fillna(method='ffill') diff --git a/app/reports/ActivityWinrateReport.py b/app/reports/ActivityWinrateReport.py index 77b45fb..da3d0eb 100644 --- a/app/reports/ActivityWinrateReport.py +++ b/app/reports/ActivityWinrateReport.py @@ -11,8 +11,8 @@ def save(self): def getName(self) -> str: return "[PVP] chart_tree - activity winrate; per mode and map" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def generate(self, data) -> Report: @@ -30,18 +30,19 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm typ = [] mode = [] directorActivity = [] wintype = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue - if datapoint["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if datapoint["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "PvP" elif datapoint["activityDetails"]["mode"] in [75, 63]: typus = "Gambit" @@ -73,8 +74,6 @@ def generateData(self, data): else: directorActivity.append(key) - print("done") - df = pd.DataFrame({ "type": typ, "mode": mode, diff --git a/app/reports/FireteamActivityReport.py b/app/reports/FireteamActivityReport.py index 3c8df5b..c30d7e0 100644 --- a/app/reports/FireteamActivityReport.py +++ b/app/reports/FireteamActivityReport.py @@ -7,15 +7,14 @@ class FireteamActivityReport(Report): def save(self): - with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - fireteam member activities"), "w", encoding='utf-8') as f: - self.df.to_csv(f, index=False, line_terminator='\n') - print("Report> Generated %s" % self.getName()) + with open("%s/%s.csv" % (Director.GetResultDirectory(self.displayName), "[ALL] table - fireteam member activities"), "w", encoding='utf-8') as f: + self.df.to_csv(f, index=False) def getName(self) -> str: return "[ALL] table - fireteam member activities" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.df = None def generate(self, data) -> Report: @@ -23,10 +22,13 @@ def generate(self, data) -> Report: return self def generateListDataframe(self, datap): + from tqdm import tqdm + eps = [] displayNames = dict() displayNameTimes = dict() - for data in datap: + + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue # find own user entry entry = [e for e in data["entries"] if e["player"]["destinyUserInfo"]["membershipId"] == str(self.membershipId)][0] diff --git a/app/reports/FireteamRace.py b/app/reports/FireteamRace.py index 496187f..90f2265 100644 --- a/app/reports/FireteamRace.py +++ b/app/reports/FireteamRace.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] race - fireteam playtime" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def save(self): @@ -25,7 +25,7 @@ def generate(self, data) -> Report: bcr.bar_chart_race( df=df, - filename=Director.GetResultDirectory(self.membershipType, self.membershipId) + "/" + self.getName() + "." + self.video_type, + filename=Director.GetResultDirectory(self.displayName, ) + "/" + self.getName() + "." + self.video_type, orientation='h', sort='desc', n_bars=20, @@ -49,11 +49,13 @@ def generate(self, data) -> Report: return self def generateData(self, datap): + from tqdm import tqdm + eps = [] displayNames = dict() displayNameTimes = dict() - for data in datap: + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue # find own user entry entry = [e for e in data["entries"] if e["player"]["destinyUserInfo"]["membershipId"] == str(self.membershipId)][0] @@ -89,9 +91,9 @@ def generateData(self, datap): eps += res - df = pd.DataFrame(eps, columns=["date", "name", "minutes"]) - df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum().reset_index() - df2["cumsum"] = df2.groupby(["name"]).cumsum() + df = pd.DataFrame(data=eps, columns=["date", "name", "minutes"]) + df2 = df.groupby(by=[df.date.dt.to_period(freq='W'), "name"]).sum(numeric_only=True).reset_index() + df2["cumsum"] = df2.groupby(["name"]).cumsum(numeric_only=True) df3 = df2.pivot(index="date", columns="name", values="cumsum") df3 = df3.fillna(method='ffill') diff --git a/app/reports/KDReport.py b/app/reports/KDReport.py index dc6106a..45fb082 100644 --- a/app/reports/KDReport.py +++ b/app/reports/KDReport.py @@ -14,8 +14,8 @@ def save(self): def getName(self) -> str: return "[PVP] chart_line - kd and kda ratio" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) @@ -39,6 +39,8 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] starttime_str = [] endtime_str = [] @@ -49,7 +51,7 @@ def generateRawDataframe(self, data): deaths = [] kills = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: @@ -80,7 +82,7 @@ def generateRawDataframe(self, data): "deaths": deaths, "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 return df diff --git a/app/reports/KillsDeathsAssistsReport.py b/app/reports/KillsDeathsAssistsReport.py index 699c3ef..f25a879 100644 --- a/app/reports/KillsDeathsAssistsReport.py +++ b/app/reports/KillsDeathsAssistsReport.py @@ -14,8 +14,8 @@ def save(self): def getName(self) -> str: return "[PVP] chart_bar - kills, deaths and assists per week" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) @@ -40,6 +40,8 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] starttime_str = [] endtime_str = [] @@ -49,7 +51,7 @@ def generateRawDataframe(self, data): deaths = [] kills = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: @@ -78,7 +80,7 @@ def generateRawDataframe(self, data): "assists": assists, }) - df["is_pvp"] = df["mode"].astype("int32").isin([84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 + df["is_pvp"] = df["mode"].astype("int32").isin([92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]) * 1 df['Date'] = pd.to_datetime(df['start']) - pd.to_timedelta(7, unit='d') return df diff --git a/app/reports/LightLevelReport.py b/app/reports/LightLevelReport.py index f0b842e..bed6a61 100644 --- a/app/reports/LightLevelReport.py +++ b/app/reports/LightLevelReport.py @@ -16,8 +16,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_line - lightlevel" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) @@ -36,11 +36,13 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] endtime = [] lightlevel = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/PlaytimeCharacterReport.py b/app/reports/PlaytimeCharacterReport.py index 24d675d..b9688bb 100644 --- a/app/reports/PlaytimeCharacterReport.py +++ b/app/reports/PlaytimeCharacterReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_area - playtime, split per character" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) @@ -40,6 +40,7 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm starttime_str = [] starttime = [] @@ -47,7 +48,7 @@ def generateRawDataframe(self, data): clazz = [] character = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/PlaytimeReport.py b/app/reports/PlaytimeReport.py index 77e465a..5d40559 100644 --- a/app/reports/PlaytimeReport.py +++ b/app/reports/PlaytimeReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_bar - playtime, weekly and daily" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateDataframe(data) @@ -35,12 +35,14 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] endtime = [] playtime = [] mode = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/ReportBase.py b/app/reports/ReportBase.py index 3fb132d..23c8ed6 100644 --- a/app/reports/ReportBase.py +++ b/app/reports/ReportBase.py @@ -6,10 +6,11 @@ class Report: - def __init__(self, membershipType, membershipId, manifest: DestinyManifest) -> None: + def __init__(self, membershipType, membershipId, displayName, manifest: DestinyManifest) -> None: super().__init__() self.membershipType = membershipType self.membershipId = membershipId + self.displayName = displayName self.manifest = manifest self.fig = None @@ -22,9 +23,8 @@ def getName(self) -> str: return "unnamed" def save(self): + print("Report> Saving %s" % self.getName()) assert self.fig is not None - filename = '%s/%s.html' % (Director.GetResultDirectory(self.membershipType, self.membershipId), self.getName()) + filename = '%s/%s.html' % (Director.GetResultDirectory(self.displayName), self.getName()) with open(filename, "w") as f: f.write(self.fig.to_html(full_html=False, include_plotlyjs='cdn')) - - print("Report> Generated %s" % self.getName()) diff --git a/app/reports/WeaponKillTreeReport.py b/app/reports/WeaponKillTreeReport.py index 6c6aa4b..19cccda 100644 --- a/app/reports/WeaponKillTreeReport.py +++ b/app/reports/WeaponKillTreeReport.py @@ -10,8 +10,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_tree - weapon kills; by type" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: df = self.generateData(data) @@ -27,12 +27,14 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm + weaponType = [] weapon = [] killType = [] kills = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue for entry in datapoint["entries"]: if entry["player"]["destinyUserInfo"]["membershipId"] != str(self.membershipId): continue diff --git a/app/reports/WeaponRaceReport.py b/app/reports/WeaponRaceReport.py index a1b7ff1..ccef723 100644 --- a/app/reports/WeaponRaceReport.py +++ b/app/reports/WeaponRaceReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] race - weapon usage" - def __init__(self, membershipType, membershipId, manifest, video_type="gif") -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest, video_type="gif") -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.video_type = video_type def save(self): @@ -30,7 +30,7 @@ def generateintern(self, data, type): df = self.generateData(data, type) bcr.bar_chart_race( df=df, - filename=Director.GetResultDirectory(self.membershipType, self.membershipId) + "/" + filename=Director.GetResultDirectory(self.displayName) + "/" + "[" + type.upper() + "] race - weapon usage." + self.video_type, orientation='h', sort='desc', @@ -56,9 +56,11 @@ def generateintern(self, data, type): ) def generateData(self, datap, typ="pve"): # pve, pvp, gambit + from tqdm import tqdm + eps = [] - for data in datap: + for data in tqdm(datap, desc=self.getName()): if "entries" not in data: continue date = parser.parse(data["period"]) # find own user entry @@ -66,7 +68,7 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit if "weapons" not in entry["extended"]: continue typus = "pve" - if data["activityDetails"]["mode"] in [84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: + if data["activityDetails"]["mode"] in [92, 91, 90, 89, 84, 81, 80, 74, 73, 72, 71, 68, 65, 62, 61, 60, 59, 50, 48, 32, 43, 45, 44, 41, 42, 37, 38, 31, 25, 15]: typus = "pvp" #elif data["activityDetails"]["mode"] in [75, 63]: # typus = "gambit" if typus != typ: @@ -85,8 +87,8 @@ def generateData(self, datap, typ="pve"): # pve, pvp, gambit ) df = pd.DataFrame(eps, columns=["date", "name", "kills"]) - df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum().reset_index() - df2["cumsum"] = df2.groupby(["name"]).cumsum() + df2 = df.groupby([df.date.dt.to_period('W'), "name"]).sum(numeric_only=True).reset_index() + df2["cumsum"] = df2.groupby(["name"]).cumsum(numeric_only=True) df3 = df2.pivot(index="date", columns="name", values="cumsum") df3 = df3.fillna(method='ffill') diff --git a/app/reports/WeaponReport.py b/app/reports/WeaponReport.py index 7976d1b..3d8e098 100644 --- a/app/reports/WeaponReport.py +++ b/app/reports/WeaponReport.py @@ -13,21 +13,21 @@ class WeaponReport(Report): def save(self): - with open("%s/%s.html" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - weapons used per activity type"), "w") as f: + with open("%s/%s.html" % (Director.GetResultDirectory(self.displayName), "[ALL] table - weapons used per activity type"), "w") as f: f.write("This table shows every weapon you ever used - in any activity. " "Sort order: Type > Weapon Name > Weapon ID > Kills." "
Generated by Mijago") f.write(build_table(self.df, 'blue_light')) - with open("%s/%s.csv" % (Director.GetResultDirectory(self.membershipType, self.membershipId), "[ALL] table - weapons used per activity type"), "w") as f: - self.df.to_csv(f, index=False, line_terminator='\n') + with open("%s/%s.csv" % (Director.GetResultDirectory(self.displayName), "[ALL] table - weapons used per activity type"), "w") as f: + self.df.to_csv(f, index=False) super().save() def getName(self) -> str: return "[ALL] chart_bar - weapons used over time; per activity type " - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) self.df = None def generate(self, data) -> Report: @@ -51,6 +51,8 @@ def generate(self, data) -> Report: return self def generateRawDataframe(self, data): + from tqdm import tqdm + starttime = [] endtime = [] weapon = [] @@ -62,7 +64,7 @@ def generateRawDataframe(self, data): mode_name = [] kills_precision = [] - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/app/reports/WeekdayReport.py b/app/reports/WeekdayReport.py index e7f55bd..286beb5 100644 --- a/app/reports/WeekdayReport.py +++ b/app/reports/WeekdayReport.py @@ -13,8 +13,8 @@ def save(self): def getName(self) -> str: return "[ALL] chart_heat - playtime per weekday" - def __init__(self, membershipType, membershipId, manifest) -> None: - super().__init__(membershipType, membershipId, manifest) + def __init__(self, membershipType, membershipId, displayName, manifest) -> None: + super().__init__(membershipType, membershipId, displayName, manifest) def generate(self, data) -> Report: timeArray = self.generateData(data) @@ -43,6 +43,8 @@ def generate(self, data) -> Report: return self def generateData(self, data): + from tqdm import tqdm + timeArray = [[0 for k in range(0, 24)] for n in range(0, 7)] lookup = set() @@ -54,7 +56,7 @@ def generateKey(xdate): xdate.hour ) - for datapoint in data: + for datapoint in tqdm(data, desc=self.getName()): if "entries" not in datapoint: continue timestamp = dateutil.parser.parse(datapoint["period"]).timestamp() for entry in datapoint["entries"]: diff --git a/main.py b/main.py index f2e1c26..5e5f90d 100644 --- a/main.py +++ b/main.py @@ -19,56 +19,88 @@ from app.reports.WeaponRaceReport import WeaponRaceReport from app.reports.WeaponReport import WeaponReport from app.reports.WeekdayReport import WeekdayReport -# from app.DiscordSender import DiscordSender +############################################################################### +# +# main() +# +############################################################################### if __name__ == '__main__': - import pathos - from pathos.multiprocessing import ProcessPool - pathos.helpers.freeze_support() # required for windows - manifest = DestinyManifest().update() + import pathos, argparse, os + + # build argument parsing + descriptionString = """Get and compile stats for a Destiny 2 user. + example: main.py -p 3 -id 4611686018472661350""" + platformString = """ Xbox 1 + Psn 2 + Steam 3 + Blizzard 4 + Stadia 5 + Egs 6""" + parser = argparse.ArgumentParser(prog='main.py', description=f'{descriptionString}', formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--platform', '-p', type=int, required=False, help=f'{platformString}') + parser.add_argument('--membership_id', '-id', type=int, required=False, help='Bungie ID') + args = vars(parser.parse_args()) + platform = args['platform'] + id = args['membership_id'] + + if platform != None and id != None: + USED_MEMBERSHIP = (platform, id) + else: + MIJAGO = (3, 4611686018482684809) + # You can easily set your own ID here: + MYCOOLID = (3, 1234567890123456789) + USED_MEMBERSHIP = MIJAGO - pool = ProcessPool(25) - # You could also specify the amount of threads. Not that this DRASTICALLY speeds up the process, but takes serious computation power - # pool = ProcessPool(60) + from pathos.multiprocessing import ProcessPool, ThreadPool, ThreadingPool + pathos.helpers.freeze_support() # required for windows + pool = ProcessPool() + # You could also specify the amount of threads. Note that this DRASTICALLY speeds up the process but takes serious computation power. + # pool = ProcessPool(128) - MEMBERSHIP_MIJAGO = (3, 4611686018482684809) - USED_MEMBERSHIP = MEMBERSHIP_MIJAGO + # check manifest + manifest = DestinyManifest().update() - api = BungieApi("API-KEY") - VIDEO_TYPE = "gif" # you can also use "mp4" if you installed ffmpeg; see README.d + # You can also set an api key manually, if you do not want to use environment variables. + API_KEY = os.getenv('BUNGIE_API_KEY') + # API_KEY = "123456789" + + api = BungieApi(API_KEY) + # "mp4" if you installed ffmpeg which you should; see README.d. otherwise "gif" if you do not. + VIDEO_TYPE = "mp4" - Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) - Director.ClearResultDirectory(*USED_MEMBERSHIP) - Director.CreateDirectoriesForUser(*USED_MEMBERSHIP) pc = PGCRCollector(*USED_MEMBERSHIP, api, pool) - pc.getCharacters().getActivities(limit=None).getPGCRs(pagesize=1000) # .combineAllPgcrs() + displayName = pc.getProfile().getDisplayName() + + Director.CreateDirectoriesForUser(displayName) + Director.ClearResultDirectory(displayName) + Director.CreateDirectoriesForUser(displayName) + + pc.getCharacters().getActivities(limit=None).getPGCRs() # .combineAllPgcrs() data = pc.getAllPgcrs() pool.close() reports = [ - KDReport(*USED_MEMBERSHIP, manifest), - KillsDeathsAssistsReport(*USED_MEMBERSHIP, manifest), - WeaponReport(*USED_MEMBERSHIP, manifest), - LightLevelReport(*USED_MEMBERSHIP, manifest), - PlaytimeReport(*USED_MEMBERSHIP, manifest), - PlaytimeCharacterReport(*USED_MEMBERSHIP, manifest), - ActivityCountReport(*USED_MEMBERSHIP, manifest), - WeekdayReport(*USED_MEMBERSHIP, manifest), - ActivityLocationTimeReport(*USED_MEMBERSHIP, manifest), - ActivityLocationWeaponReport(*USED_MEMBERSHIP, manifest), - ActivityWinrateReport(*USED_MEMBERSHIP, manifest), - WeaponKillTreeReport(*USED_MEMBERSHIP, manifest), - FireteamRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - WeaponRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - ActivityTypeRaceReport(*USED_MEMBERSHIP, manifest, video_type=VIDEO_TYPE), - FireteamActivityReport(*USED_MEMBERSHIP, manifest) + ActivityCountReport(*USED_MEMBERSHIP, displayName, manifest), + ActivityLocationTimeReport(*USED_MEMBERSHIP, displayName, manifest), + ActivityLocationWeaponReport(*USED_MEMBERSHIP, displayName, manifest), + ActivityTypeRaceReport(*USED_MEMBERSHIP, displayName, manifest, video_type=VIDEO_TYPE), + ActivityWinrateReport(*USED_MEMBERSHIP, displayName, manifest), + FireteamActivityReport(*USED_MEMBERSHIP, displayName, manifest), + FireteamRaceReport(*USED_MEMBERSHIP, displayName, manifest, video_type=VIDEO_TYPE), + KDReport(*USED_MEMBERSHIP, displayName, manifest), + KillsDeathsAssistsReport(*USED_MEMBERSHIP, displayName, manifest), + LightLevelReport(*USED_MEMBERSHIP, displayName, manifest), + PlaytimeCharacterReport(*USED_MEMBERSHIP, displayName, manifest), + PlaytimeReport(*USED_MEMBERSHIP, displayName, manifest), + WeaponKillTreeReport(*USED_MEMBERSHIP, displayName, manifest), + WeaponRaceReport(*USED_MEMBERSHIP, displayName, manifest, video_type=VIDEO_TYPE), + WeaponReport(*USED_MEMBERSHIP, displayName, manifest), + WeekdayReport(*USED_MEMBERSHIP, displayName, manifest) ] for report in reports: report.generate(data).save() - Zipper.zip_directory(Director.GetResultDirectory(*USED_MEMBERSHIP), Director.GetZipPath(*USED_MEMBERSHIP)) - print("Generated ZIP:", Director.GetZipPath(*USED_MEMBERSHIP)) - - # DiscordSender.send(Director.GetZipPath(*USED_MEMBERSHIP), *USED_MEMBERSHIP) - # print("Sent ZIP:", Director.GetZipPath(*USED_MEMBERSHIP)) + Zipper.zip_directory(Director.GetResultDirectory(displayName), Director.GetZipPath(displayName)) + print("Generated ZIP:", Director.GetZipPath(displayName))