diff --git a/README.md b/README.md index 8c4a339..006cc52 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,16 @@ Import/Export Mods from Tabletop Simulator, including all assets. ## Status - -Currently this code is rather alpha quality. It has also only been tested on a limited number of mods and machines. **Do not rely on this to backup your files without checking it restores correctly on another install.** If you find a configuration / mod that doesn't work, please let me know. +Currently this code is rather alpha quality. It has also only been tested on a limited number of mods and machines. +**Do not rely on this to backup your files without checking it restores correctly on another install.** +If you find a configuration / mod that doesn't work, please let me know. Listing, export and import all should work. Note that old-style mods (`.cjc` files) are *not* supported - they simply will not be listed. -To export a mod, you ideally should have downloaded *all* assets. Opening a mod in Tabletop Simulator is usually enough, but make sure you have taken something out of every bag in the mod. If anything is missing, then the tool will tell you. TTS Manager can attempt to download the files for you, but this feature is very new. +To export a mod, you ideally should have downloaded *all* assets. +Opening a mod in Tabletop Simulator is usually enough, but make sure you have taken something out of every bag in the mod. +If anything is missing, then the tool will tell you. +TTS Manager can attempt to download the files for you, but this feature is very new. ## Quickstart Download the installer from the [releases](https://github.com/cwoac/TTS-Manager/releases) and install it. Then run the gui from the created shortcut link. @@ -26,3 +30,9 @@ These are primarily tracked on github, but roughly: - A better gui - downloading arbitary pak files - LOTS MORE TESTING. + +## Changelog +* V0.6.0 + - Now correctly grabs non-image/model types for the new layout of TTS mods + - Added ability to export just the 'unavailable' files of a mod to a 'part.pak' file. + - Various bug fixes and preliminary Linux/Mac support from 'bobpaul' (thanks!) \ No newline at end of file diff --git a/tts/filesystem.py b/tts/filesystem.py index 4cb8635..50afd6b 100644 --- a/tts/filesystem.py +++ b/tts/filesystem.py @@ -1,172 +1,164 @@ import os import os.path -import tts import platform -if platform.system() == 'Linux': - import xdgappdirs +from typing import Tuple -def standard_basepath(): - if platform.system() == 'Windows': - basepath = os.path.join(os.path.expanduser("~"),"Documents","My Games","Tabletop Simulator") - elif platform.system() == 'Linux': - basepath = os.path.join(xdgappdirs.user_data_dir(),"Tabletop Simulator") - else: - basepath = os.path.join(os.path.expanduser("~"),"Library","Tabletop Simulator") - return basepath - -class FileSystem: - def __init__(self,base_path=None,tts_install_path=None): - if base_path is not None: - self.basepath=base_path - else: - self.basepath=standard_basepath() - if tts_install_path is not None: - self.modpath=os.path.join(tts_install_path,"Tabletop Simulator_Data") - else: - self.modpath=self.basepath - self._saves = os.path.join(self.basepath,"Saves") - self._chest = os.path.join(self._saves,"Chest") - self._mods = os.path.join(self.modpath,"Mods") - self._images= os.path.join(self._mods,"Images") - self._models= os.path.join(self._mods,"Models") - self._workshop = os.path.join(self._mods,"Workshop") - - def get_dir_by_type(self,save_type): - st={ - tts.SaveType.workshop:self._workshop, - tts.SaveType.save:self._saves, - tts.SaveType.chest:self._chest - } - return st[save_type] - - def check_dirs(self): - """Do all the directories exist?""" - for dir in [ self._saves, self._mods, self._images, self._models, self._workshop ]: - if not os.path.isdir(dir): - tts.logger().error("TTS Dir missing: {}".format(dir)) - return False - #These directories don't always exist, and that's OK - for dir in [ self._chest ]: - if not os.path.isdir(dir): - tts.logger().warn("TTS Dir missing: {}".format(dir)) - return True - - def create_dirs(self): - """Attempt to create any missing directories.""" - for dir in [ self._saves, self._chest, self._mods, self._images, self._models, self._workshop ]: - os.makedirs(dir,exist_ok=True) - - @property - def saves_dir(self): - return self._saves - - @property - def images_dir(self): - return self._images - - def get_image_path(self,filename): - return os.path.join(self._images,filename) - - def get_model_path(self,filename): - return os.path.join(self._models,filename) - - def get_workshop_path(self,filename): - return os.path.join(self._workshop,filename) - - def get_save_path(self,filename): - return os.path.join(self._saves,filename) - - def get_chest_path(self,filename): - return os.path.join(self._chest,filename) - - def get_path_by_type(self,filename,save_type): - return os.path.join(self.get_dir_by_type(save_type),filename) - - def find_details(self,basename): - result=self.find_image(basename) - if result: - return result,True - result=self.find_model(basename) - if result: - return result,False - return None,None - - def find_image(self,basename): - result=None - stripname = tts.strip_filename(basename) - for image_format in ['.png','.jpg','.bmp']: - filename=os.path.join(self._images,stripname+image_format) - if os.path.isfile(filename): - result=filename - break - return result +import tts +import tts.util +from tts.filetype import FileType - def find_model(self,basename): - result=None - stripname = tts.strip_filename(basename) - for model_format in ['.obj']: - filename=os.path.join(self._models,stripname+model_format) - if os.path.isfile(filename): - result=filename - break - return result +if platform.system() == 'Linux': + import xdgappdirs - def get_filenames_in(self,search_path): - if not os.path.isdir(search_path): - tts.logger().warn("Tried to search non-existent path {}.".format(search_path)) - return [] - return [os.path.splitext(file)[0] for file in os.listdir(search_path) if os.path.splitext(file)[1].lower()=='.json'] - - def get_save_filenames(self): - files=self.get_filenames_in(self._saves) - if files and 'SaveFileInfos' in files: - files.remove('SaveFileInfos') - return files - - def get_workshop_filenames(self): - files=self.get_filenames_in(self._workshop) - if files and 'WorkshopFileInfos' in files: - files.remove('WorkshopFileInfos') - return files - - def get_chest_filenames(self): - return self.get_filenames_in(self._chest) - - def get_filenames_by_type(self,save_type): - if save_type==tts.SaveType.workshop: - return self.get_workshop_filenames() - if save_type==tts.SaveType.save: - return self.get_save_filenames() - if save_type==tts.SaveType.chest: - return self.get_chest_filenames() - # TODO: error handling here - return None - def get_json_filename_from(self,basename,paths): - result=None +def get_json_filename_from(basename, paths): + result = None for pth in paths: - filename=os.path.join(pth,basename+'.json') - if os.path.isfile(filename): - result=filename - break + filename = os.path.join(pth, basename + '.json') + if os.path.isfile(filename): + result = filename + break # TODO: error handling here return result - def get_json_filename(self,basename): - return self.get_json_filename_from(basename,[self._workshop,self._saves,self._chest]) - def get_json_filename_for_type(self,basename,save_type): - return self.get_json_filename_from(basename,[self.get_dir_by_type(save_type)]) +def get_filenames_in(search_path): + if not os.path.isdir(search_path): + tts.logger().warn("Tried to search non-existent path {}.".format(search_path)) + return [] + return [os.path.splitext(file)[0] for file in os.listdir(search_path) if + os.path.splitext(file)[1].lower() == '.json'] - def get_json_filename_type(self,basename): - if os.path.isfile(os.path.join(self._workshop,basename+'.json')): - return tts.SaveType.workshop - if os.path.isfile(os.path.join(self._saves,basename+'.json')): - return tts.SaveType.save - if os.path.isfile(os.path.join(self._chest,basename+'.json')): - return tts.SaveType.chest - # TODO: error handling here - return None - def __str__(self): - return "Saves: {} Mods: {}".format(self.basepath,self.modpath) +def standard_basepath(): + if platform.system() == 'Windows': + basepath = os.path.join(os.path.expanduser("~"), "Documents", "My Games", "Tabletop Simulator") + elif platform.system() == 'Linux': + basepath = os.path.join(xdgappdirs.user_data_dir(), "Tabletop Simulator") + else: + basepath = os.path.join(os.path.expanduser("~"), "Library", "Tabletop Simulator") + return basepath + + +class FileSystem: + def __init__(self, base_path=None, tts_install_path=None): + if base_path is not None: + self.basepath = base_path + else: + self.basepath = standard_basepath() + if tts_install_path is not None: + self.modpath = os.path.join(tts_install_path, "Tabletop Simulator_Data") + else: + self.modpath = self.basepath + self._saves = os.path.join(self.basepath, "Saves") + self._chest = os.path.join(self._saves, "Chest") + self._mods = os.path.join(self.modpath, "Mods") + self._images = os.path.join(self._mods, "Images") + self._models = os.path.join(self._mods, "Models") + self._workshop = os.path.join(self._mods, "Workshop") + + def get_dir_by_type(self, save_type): + st = { + tts.SaveType.workshop: self._workshop, + tts.SaveType.save: self._saves, + tts.SaveType.chest: self._chest + } + return st[save_type] + + def check_dirs(self): + """Do all the directories exist?""" + for dir in [self._saves, self._mods, self._images, self._models, self._workshop]: + if not os.path.isdir(dir): + tts.logger().error("TTS Dir missing: {}".format(dir)) + return False + # These directories don't always exist, and that's OK + for dir in [self._chest]: + if not os.path.isdir(dir): + tts.logger().warn("TTS Dir missing: {}".format(dir)) + return True + + def create_dirs(self): + """Attempt to create any missing directories.""" + for dir in [self._saves, self._chest, self._mods, self._images, self._models, self._workshop]: + os.makedirs(dir, exist_ok=True) + + @property + def saves_dir(self): + return self._saves + + @property + def images_dir(self): + return self._images + + def get_dir(self, type: FileType) -> str: + return os.path.join(self._mods, type.value) + + def get_file_path(self, file_name: str, file_type: FileType) -> str: + return os.path.join(self.get_dir(file_type), file_name) + + def get_path_by_save_type(self, filename, save_type): + return os.path.join(self.get_dir_by_type(save_type), filename) + + def check_for_file_location(self, basename: str, type: FileType) -> Tuple[str, str]: + if type is FileType.IMAGE: + return self.find_image(basename) + if type is FileType.NONE: + return None + extension = type.get_extension(None) + filename = os.path.join(self.get_dir(type), + f"{tts.util.strip_filename(basename)}{extension}") + return (filename, extension) if os.path.isfile(filename) else None + + def find_image(self, basename: str) -> str: + result = None + stripname = tts.util.strip_filename(basename) + for image_format in ['.png', '.jpg', '.bmp']: + filename = os.path.join(self._images, stripname + image_format) + if os.path.isfile(filename): + result = filename, image_format + break + return result + + def get_save_filenames(self): + files = get_filenames_in(self._saves) + if files and 'SaveFileInfos' in files: + files.remove('SaveFileInfos') + return files + + def get_workshop_filenames(self): + files = get_filenames_in(self._workshop) + if files and 'WorkshopFileInfos' in files: + files.remove('WorkshopFileInfos') + return files + + def get_chest_filenames(self): + return get_filenames_in(self._chest) + + def get_filenames_by_type(self, save_type): + if save_type == tts.SaveType.workshop: + return self.get_workshop_filenames() + if save_type == tts.SaveType.save: + return self.get_save_filenames() + if save_type == tts.SaveType.chest: + return self.get_chest_filenames() + # TODO: error handling here + return None + + def get_json_filename(self, basename): + return get_json_filename_from(basename, [self._workshop, self._saves, self._chest]) + + def get_json_filename_for_type(self, basename, save_type): + return get_json_filename_from(basename, [self.get_dir_by_type(save_type)]) + + def get_json_filename_type(self, basename): + if os.path.isfile(os.path.join(self._workshop, basename + '.json')): + return tts.SaveType.workshop + if os.path.isfile(os.path.join(self._saves, basename + '.json')): + return tts.SaveType.save + if os.path.isfile(os.path.join(self._chest, basename + '.json')): + return tts.SaveType.chest + # TODO: error handling here + return None + + def __str__(self): + return "Saves: {} Mods: {}".format(self.basepath, self.modpath) diff --git a/tts/filetype.py b/tts/filetype.py new file mode 100644 index 0000000..715acbc --- /dev/null +++ b/tts/filetype.py @@ -0,0 +1,63 @@ +from enum import Enum + +from tts.util import identify_image_extension + + +class FileType(Enum): + ASSETBUNDLE = "Assetbundles" + AUDIO = "Audio" + IMAGE = "Images" + MODEL = "Models" + PDF = "PDF" + TEXT = "Text" + WORKSHOP = "Workshop" + NONE = "unk" + + @classmethod + def identify_type(cls, url_key: str): + result = None + try: + result = _urlkeys[url_key] + except KeyError: + pass + return result + + def get_extension(self, data): + if self is FileType.IMAGE: + image_type = identify_image_extension(data) + if image_type: + return image_type + return "" + return _url_extensions[self] + + +_urlkeys = { + "AssetbundleURL": FileType.ASSETBUNDLE, + "AssetbundleSecondaryURL": FileType.ASSETBUNDLE, + "BackURL": FileType.IMAGE, + "ColliderURL": FileType.MODEL, + "CurrentAudioURL": FileType.AUDIO, + "DiffuseURL": FileType.IMAGE, + "FaceURL": FileType.IMAGE, + "ImageSecondaryURL": FileType.IMAGE, + "ImageURL": FileType.IMAGE, + "LutURL": FileType.NONE, # TODO:: Figure out where lookup tables go + "MeshURL": FileType.MODEL, + "NormalURL": FileType.IMAGE, + "PageURL": FileType.NONE, # Not downloading this one + "PDFUrl": FileType.PDF, + "SkyURL": FileType.IMAGE, + "TableURL": FileType.IMAGE, + "URL": FileType.IMAGE +} + +# Unfortunately Images can be png or jpg. I suspect Audio may be the same, +# but all the files I have are MP3 +_url_extensions = { + FileType.ASSETBUNDLE: ".unity3d", + FileType.AUDIO: ".MP3", + FileType.MODEL: ".obj", + FileType.PDF: ".pdf", + FileType.TEXT: ".txt", + FileType.NONE: "" +} diff --git a/tts/save.py b/tts/save.py index 3b09efa..839cf9e 100644 --- a/tts/save.py +++ b/tts/save.py @@ -1,223 +1,264 @@ +import zipfile +from typing import Tuple, Set + +import tts +from tts.filetype import FileType from .tts import * from .url import Url -import tts -import zipfile -import json -import urllib.error - -PAK_VER=2 - -def importPak(filesystem,filename): - log=tts.logger() - log.debug("About to import {} into {}.".format(filename,filesystem)) - if not os.path.isfile(filename): - log.error("Unable to find mod pak {}".format(filename)) - return False - if not zipfile.is_zipfile(filename): - log.error("Mod pak {} format appears corrupt.".format(filename)) - return False - try: - with zipfile.ZipFile(filename,'r') as zf: - bad_file=zf.testzip() - if bad_file: - log.error("At least one corrupt file found in {} - {}".format(filename,bad_file)) - return False - if not zf.comment: - # TODO: allow overrider - log.error("Missing pak header comment in {}. Aborting import.".format(filename)) + +PAK_VER = 3 + + +def import_pak(filesystem, filename): + log = tts.logger() + log.debug("About to import {} into {}.".format(filename, filesystem)) + if not os.path.isfile(filename): + log.error("Unable to find mod pak {}".format(filename)) return False - metadata=json.loads(zf.comment.decode('utf-8')) - if not tts.validate_metadata(metadata, PAK_VER): - log.error(f"Invalid pak header '{metadata}' in {filename}. Aborting import.") + if not zipfile.is_zipfile(filename): + log.error("Mod pak {} format appears corrupt.".format(filename)) return False - log.info(f"Extracting {metadata['Type']} pak for id {metadata['Id']} (pak version {metadata['Ver']})") - - #select the thumbnail which matches the metadata id, else anything - names = zf.namelist() - thumbnails = [name for name in names if '/Thumbnails/' in name] - thumbnail = None - for thumbnail in thumbnails: - if metadata['Id'] in os.path.basename(thumbnail): - break - - outname=None - for name in names: - # Note that zips always use '/' as the seperator it seems. - splitname = name.split('/') - if len(splitname) > 2 and splitname[2] == 'Thumbnails': - if name == thumbnail: - #remove "Thumbnails" from the path - outname='/'.join(splitname[0:2] + [os.path.extsep.join([metadata['Id'],'png'])]) - else: - outname=None - continue - - start=splitname[0] - if start=='Saves': - modpath=filesystem.basepath - else: - modpath=filesystem.modpath - log.debug(f"Extracting {name} to {modpath}") - zf.extract(name,modpath) - if outname: - log.debug(f"Renaming {name} to {outname}") - os.rename(os.path.join(modpath,name), os.path.join(modpath,outname)) - try: - outdir = os.path.dirname(os.path.join(modpath,name)) - os.rmdir(outdir) - except OSError: - log.debug(f"Can't remove dir {outdir}") - - except zipfile.BadZipFile as e: - log.error("Mod pak {} format appears corrupt - {}.".format(filename,e)) - except zipfile.LargeZipFile as e: - log.error("Mod pak {} requires large zip capability - {}.\nThis shouldn't happen - please raise a bug.".format(filename,e)) - log.info("Imported {} successfully.".format(filename)) - return True - -def get_save_urls(savedata): - ''' - Iterate over all the values in the json file, building a (key,value) set of - all the values whose key ends in "URL" - ''' - log=tts.logger() - def parse_list(data): - urls=set() - for item in data: - urls |= get_save_urls(item) - return urls - def parse_dict(data): - urls=set() - if not data: - return urls - for key in data: - if type(data[key]) is not str or key=='PageURL' or key=='Rules': - # If it isn't a string, it can't be an url. - # Also don't save tablet state / rulebooks - continue - if key.endswith('URL') and data[key]!='': - log.debug("Found {}:{}".format(key,data[key])) - urls.add(data[key]) - continue - protocols=data[key].split('://') - if len(protocols)==1: - # not an url - continue - if protocols[0] in ['http','https','ftp']: - # belt + braces. - urls.add(data[key]) - log.debug("Found {}:{}".format(key,data[key])) - continue - for item in data.values(): - urls |= get_save_urls(item) - return urls - - if type(savedata) is list: - return parse_list(savedata) - if type(savedata) is dict: - return parse_dict(savedata) - return set() + try: + with zipfile.ZipFile(filename, 'r') as zf: + bad_file = zf.testzip() + if bad_file: + log.error("At least one corrupt file found in {} - {}".format(filename, bad_file)) + return False + if not zf.comment: + # TODO: allow overrider + log.error("Missing pak header comment in {}. Aborting import.".format(filename)) + return False + metadata = json.loads(zf.comment.decode('utf-8')) + if not tts.validate_metadata(metadata, PAK_VER): + log.error(f"Invalid pak header '{metadata}' in {filename}. Aborting import.") + return False + log.info(f"Extracting {metadata['Type']} pak for id {metadata['Id']} (pak version {metadata['Ver']})") + # select the thumbnail which matches the metadata id, else anything + names = zf.namelist() + thumbnails = [name for name in names if '/Thumbnails/' in name] + thumbnail = None + for thumbnail in thumbnails: + if metadata['Id'] in os.path.basename(thumbnail): + break -class Save: - def __init__(self,savedata,filename,ident,filesystem,save_type=SaveType.workshop): - log=tts.logger() - self.data = savedata - self.ident=ident - if self.data['SaveName']: - self.save_name=self.data['SaveName'] - else: - self.save_name=self.ident - self.save_type=save_type - self.filesystem = filesystem - self.filename=filename - thumbnail = os.path.extsep.join(filename.split(os.path.extsep)[0:-1] + ['png']) #Known issue: this fails if filename doesn't contain an extsep - if os.path.isfile(thumbnail): - self.thumbnail = thumbnail - else: - self.thumbnail = None - self.thumb=os.path.isfile(os.path.extsep.join([filename.split(os.path.extsep)[0],'png'])) - #strip the local part off. - fileparts=self.filename.split(os.path.sep) - while fileparts[0]!='Saves' and fileparts[0]!='Mods': - fileparts=fileparts[1:] - self.basename=os.path.join(*fileparts) - log.debug("filename: {},save_name: {}, basename: {}".format(self.filename,self.save_name,self.basename)) - self.urls = [ Url(url,self.filesystem) for url in get_save_urls(savedata) ] - self.missing = [ x for x in self.urls if not x.exists ] - self.images=[ x for x in self.urls if x.exists and x.isImage ] - self.models=[ x for x in self.urls if x.exists and not x.isImage ] - log.debug("Urls found {}:{} missing, {} models, {} images".format(len(self.urls),len(self.missing),len(self.models),len(self.images))) - - def export(self,export_filename): - log=tts.logger() - log.info("About to export %s to %s" % (self.ident,export_filename)) - zfs = tts.filesystem.FileSystem(base_path="") - zipComment = { - "Ver":PAK_VER, - "Id":self.ident, - "Type":self.save_type.name - } - - # TODO: error checking. - with zipfile.ZipFile(export_filename,'w') as zf: - zf.comment=json.dumps(zipComment).encode('utf-8') - log.debug("Writing {} (base {}) to {}".format(self.filename,os.path.basename(self.filename),zfs.get_path_by_type(os.path.basename(self.filename),self.save_type))) - zf.write(self.filename,zfs.get_path_by_type(os.path.basename(self.filename),self.save_type)) - if self.thumbnail: - filepath=zfs.get_path_by_type(os.path.basename(self.thumbnail),self.save_type) - arcname=os.path.join(os.path.dirname(filepath), 'Thumbnails', os.path.basename(filepath)) - zf.write(self.thumbnail,arcname=arcname) - log.debug(f"Writing {self.thumbnail} to {arcname}") - for url in self.models: - log.debug("Writing {} to {}".format(url.location,zfs.get_model_path(os.path.basename(url.location)))) - zf.write(url.location,zfs.get_model_path(os.path.basename(url.location))) - for url in self.images: - log.debug("Writing {} to {}".format(url.location,zfs.get_model_path(os.path.basename(url.location)))) - zf.write(url.location,zfs.get_image_path(os.path.basename(url.location))) - log.info("File exported.") - - @property - def isInstalled(self): - """Is every url referenced by this save installed?""" - return len(self.missing)==0 - - def download(self): - log=tts.logger() - log.warn("About to download files for %s" % self.save_name) - if self.isInstalled==True: - log.info("All files already downloaded.") - return True - - successful=True - url_counter=1 - for url in self.missing: - log.warn("Downloading file {} of {} for {}".format(url_counter,len(self.missing),self.save_name)) - result = url.download() - if not result: - successful=False - url_counter+=1 - - #TODO:: remove items from missing list. - return successful - - - log.info("All files downloaded.") + outname = None + for name in names: + # Note that zips always use '/' as the seperator it seems. + splitname = name.split('/') + if len(splitname) > 2 and splitname[2] == 'Thumbnails': + if name == thumbnail: + # remove "Thumbnails" from the path + outname = '/'.join(splitname[0:2] + [os.path.extsep.join([metadata['Id'], 'png'])]) + else: + outname = None + continue + + start = splitname[0] + if start == 'Saves': + modpath = filesystem.basepath + else: + modpath = filesystem.modpath + target_base_name = outname if outname else name + target_file_name = os.path.join(modpath, target_base_name) + if os.path.isfile(target_file_name): + log.warn(f"Not extracting existing file {target_file_name}") + continue + log.debug(f"Extracting {name} to {modpath}") + zf.extract(name, modpath) + if outname: + log.debug(f"Renaming {name} to {outname}") + os.rename(os.path.join(modpath, name), os.path.join(modpath, outname)) + try: + outdir = os.path.dirname(os.path.join(modpath, name)) + os.rmdir(outdir) + except OSError: + log.debug(f"Can't remove dir {outdir}") + + except zipfile.BadZipFile as e: + log.error("Mod pak {} format appears corrupt - {}.".format(filename, e)) + except zipfile.LargeZipFile as e: + log.error("Mod pak {} requires large zip capability - {}.\nThis shouldn't happen - please raise a bug.".format( + filename, e)) + log.info("Imported {} successfully.".format(filename)) return True - def __str__(self): - result = "Save: %s\n" % self.data['SaveName'] - if len(self.missing)>0: - result += "Missing:\n" - for x in self.missing: - result += str(x)+"\n" - if len(self.images)>0: - result += "Images:\n" - for x in self.images: - result += str(x)+"\n" - if len(self.models)>0: - result += "Models:\n" - for x in self.models: - result += str(x)+"\n" - return result -__all__ = [ 'Save' ] + +def get_save_urls(save_data) -> Set[Tuple[str, FileType]]: + """ + Iterate over all the values in the json file, building a (key,value) set of + all the values whose key ends in "URL" + """ + log = tts.logger() + + def parse_list(data): + urls = set() + for item in data: + urls |= get_save_urls(item) + return urls + + def parse_dict(data): + urls = set() + if not data: + return urls + for key in data: + key_type = FileType.identify_type(key) + + if not key_type or key_type is FileType.NONE: + # If it isn't a string, it can't be an url. + # Also don't save tablet state / rulebooks + continue + if data[key] != '': + log.debug(f"Found {key_type}: {key}:{data[key]}") + urls.add((data[key], key_type)) + continue + for item in data.values(): + urls |= get_save_urls(item) + return urls + + if type(save_data) is list: + return parse_list(save_data) + if type(save_data) is dict: + return parse_dict(save_data) + return set() + + +class Save: + def __init__(self, savedata, filename, ident, filesystem, save_type=SaveType.workshop): + log = tts.logger() + self.data = savedata + self.ident = ident + if self.data['SaveName']: + self.save_name = self.data['SaveName'] + else: + self.save_name = self.ident + self.save_type = save_type + self.filesystem = filesystem + self.filename = filename + thumbnail = os.path.extsep.join(filename.split(os.path.extsep)[0:-1] + [ + 'png']) # Known issue: this fails if filename doesn't contain an extsep + if os.path.isfile(thumbnail): + self.thumbnail = thumbnail + else: + self.thumbnail = None + self.thumb = os.path.isfile(os.path.extsep.join([filename.split(os.path.extsep)[0], 'png'])) + # strip the local part off. + fileparts = self.filename.split(os.path.sep) + while fileparts[0] != 'Saves' and fileparts[0] != 'Mods': + fileparts = fileparts[1:] + self.basename = os.path.join(*fileparts) + log.debug("filename: {},save_name: {}, basename: {}".format(self.filename, self.save_name, self.basename)) + self.urls = [Url(url, type, self.filesystem) for url, type in get_save_urls(savedata)] + self.missing = [x for x in self.urls if not x.exists] + self.present_files = {} + for url in [u for u in self.urls if u.exists]: + try: + self.present_files[url.type].append(url) + except KeyError: + self.present_files[url.type] = [url] + log.debug(f"Urls found {len(self.urls)} ({len(self.missing)} missing)") + + def export(self, export_filename): + log = tts.logger() + log.info("About to export %s to %s" % (self.ident, export_filename)) + zfs = tts.filesystem.FileSystem(base_path="") + zip_comment = { + "Ver": PAK_VER, + "Id": self.ident, + "Type": self.save_type.name + } + + # TODO: error checking. + with zipfile.ZipFile(export_filename, 'w') as zf: + zf.comment = json.dumps(zip_comment).encode('utf-8') + log.debug("Writing {} (base {}) to {}".format(self.filename, os.path.basename(self.filename), + zfs.get_path_by_save_type(os.path.basename(self.filename), + self.save_type))) + zf.write(self.filename, zfs.get_path_by_save_type(os.path.basename(self.filename), self.save_type)) + if self.thumbnail: + filepath = zfs.get_path_by_save_type(os.path.basename(self.thumbnail), self.save_type) + arcname = os.path.join(os.path.dirname(filepath), 'Thumbnails', os.path.basename(filepath)) + zf.write(self.thumbnail, arcname=arcname) + log.debug(f"Writing {self.thumbnail} to {arcname}") + for file_type in self.present_files: + log.info(f"Writing {file_type} files") + for url in self.present_files[file_type]: + target_file = zfs.get_file_path(os.path.basename(url.location), url.type) + log.debug(f"Writing {url.location} to {target_file}") + zf.write(url.location, target_file) + log.info("File exported.") + + def export_missing(self, export_filename): + """ Create a partial pak containing only those files currently unavailiable + """ + # TODO:: I don't like the amount of code duplication here. + # Possible options: + # 1. pass a filter function to an inner export function + # 2. Create a filtered Save subclass and use the normal export function? + log = tts.logger() + log.info(f"About to partial export {self.ident} to {export_filename}") + zfs = tts.filesystem.FileSystem(base_path="") + zip_comment = { + "Ver": PAK_VER, + "Id": self.ident, + "Type": self.save_type.name + } + + # TODO: error checking. + with zipfile.ZipFile(export_filename, 'w') as zf: + # Always write base file and thumbnail. They should be pretty small anyway. + zf.comment = json.dumps(zip_comment).encode('utf-8') + log.debug( + f"Writing {self.filename} (base {os.path.basename(self.filename)}) to {zfs.get_path_by_save_type(os.path.basename(self.filename), self.save_type)}") + zf.write(self.filename, zfs.get_path_by_save_type(os.path.basename(self.filename), self.save_type)) + if self.thumbnail: + filepath = zfs.get_path_by_save_type(os.path.basename(self.thumbnail), self.save_type) + arcname = os.path.join(os.path.dirname(filepath), 'Thumbnails', os.path.basename(filepath)) + log.debug(f"Writing {self.thumbnail} to {arcname}") + zf.write(self.thumbnail, arcname=arcname) + for file_type in self.present_files: + log.info(f"Writing {file_type} files") + for url in self.present_files[file_type]: + target_file = zfs.get_file_path(os.path.basename(url.location), url.type) + log.debug(f"Writing {url.location} to {target_file}") + zf.write(url.location, target_file) + log.info("File exported.") + + @property + def is_installed(self): + """Is every url referenced by this save installed?""" + return len(self.missing) == 0 + + def download(self): + log = tts.logger() + log.warn("About to download files for %s" % self.save_name) + if self.is_installed: + log.info("All files already downloaded.") + return True + + successful = True + url_counter = 1 + for url in self.missing: + log.warn("Downloading file {} of {} for {}".format(url_counter, len(self.missing), self.save_name)) + result = url.download() + if not result: + successful = False + url_counter += 1 + + # TODO:: remove items from missing list. + return successful + + def __str__(self): + result = "Save: %s\n" % self.data['SaveName'] + if len(self.missing) > 0: + result += "Missing:\n" + for x in self.missing: + result += str(x) + "\n" + for file_type in self.present_files: + result += f"{file_type}:\n" + for url in self.present_files[file_type]: + result += str(x) + "\n" + return result + + +__all__ = ['Save'] diff --git a/tts/tts.py b/tts/tts.py index f40a41c..8845e25 100644 --- a/tts/tts.py +++ b/tts/tts.py @@ -1,109 +1,110 @@ import os.path -import string import json import tts.logger import tts.save import codecs from enum import IntEnum -from .filesystem import FileSystem,standard_basepath +from .filesystem import FileSystem, standard_basepath + class SaveType(IntEnum): - workshop = 1 - save = 2 - chest = 3 + workshop = 1 + save = 2 + chest = 3 + def get_default_fs(): - return FileSystem(standard_basepath()) + return FileSystem(standard_basepath()) -def strip_filename(filename): - # Convert a filename to TTS format. - valid_chars = "%s%s" % (string.ascii_letters, string.digits) - return ''.join(c for c in filename if c in valid_chars) def validate_metadata(metadata, maxver): - # TODO: extract into new class - if not metadata or not isinstance(metadata, dict): - return False - return ('Ver' in metadata and metadata['Ver'] <= maxver and - 'Id' in metadata and - 'Type' in metadata and metadata['Type'] in [x.name for x in SaveType]) + # TODO: extract into new class + if not metadata or not isinstance(metadata, dict): + return False + return ('Ver' in metadata and metadata['Ver'] <= maxver and + 'Id' in metadata and + 'Type' in metadata and metadata['Type'] in [x.name for x in SaveType]) + def load_json_file(filename): - log=tts.logger() - if not filename: - log.warn("load_json_file called without filename") - return None - if not os.path.isfile(filename): - log.error("Unable to find requested file %s" % filename) - return None - log.info("loading json file %s" % filename) - encodings = ['utf-8', 'windows-1250', 'windows-1252', 'ansi'] - data=None - for encoding in encodings: - try: - data=codecs.open(filename,'r',encoding).read() - except UnicodeDecodeError as e: - log.debug("Unable to parse in encoding %s." % encoding) - else: - log.debug("loaded using encoding %s." % encoding) - break - if not data: - log.error("Unable to find encoding for %s." % filename) - return None - j_data=json.loads(data) - return j_data - -def load_file_by_type(ident,filesystem,save_type): - filename=filesystem.get_json_filename_for_type(ident,save_type) - return load_json_file(filename) + log = tts.logger() + if not filename: + log.warn("load_json_file called without filename") + return None + if not os.path.isfile(filename): + log.error("Unable to find requested file %s" % filename) + return None + log.info("loading json file %s" % filename) + encodings = ['utf-8', 'windows-1250', 'windows-1252', 'ansi'] + data = None + for encoding in encodings: + try: + data = codecs.open(filename, 'r', encoding).read() + except UnicodeDecodeError as e: + log.debug("Unable to parse in encoding %s." % encoding) + else: + log.debug("loaded using encoding %s." % encoding) + break + if not data: + log.error("Unable to find encoding for %s." % filename) + return None + j_data = json.loads(data) + return j_data + + +def load_file_by_type(ident, filesystem, save_type): + filename = filesystem.get_json_filename_for_type(ident, save_type) + return load_json_file(filename) + def describe_files_by_type(filesystem, save_type, sort_key=lambda mod: mod[0]): - """ filesystem - a filesystem object + """ filesystem - a filesystem object save_type - list only mods of type defined by SaveType enum sort_key - None or function for defining sort order. Defaults to sort by name return - List of (name, id) """ - assert isinstance(save_type, SaveType), "save_type must be a SaveType enum" - output=[] - for filename in filesystem.get_filenames_by_type(save_type): - json=load_file_by_type(filename,filesystem,save_type) - name=json['SaveName'] - output.append((name,filename)) - if sort_key: - output = sorted(output, key=sort_key) - return output - -def download_file(filesystem,ident,save_type): - """Attempt to download all files for a given savefile""" - log=tts.logger() - log.info("Downloading %s file %s (from %s)" % (save_type.name,ident,filesystem)) - filename=filesystem.get_json_filename_for_type(ident,save_type) - if not filename: - log.error("Unable to find data file.") - return False - try: - data=load_json_file(filename) - except IOError as e: - log.error("Unable to read data file %s (%s)" % (filename,e)) - return False - if not data: - log.error("Unable to read data file %s" % filename) - return False - - save=tts.Save(savedata=data, - filename=filename, - ident=ident, - save_type=save_type, - filesystem=filesystem) - - if save.isInstalled: - log.info("All files already downloaded.") - return True - - successful = save.download() - if successful: - log.info("All files downloaded.") - else: - log.info("Some files failed to download.") - return successful + assert isinstance(save_type, SaveType), "save_type must be a SaveType enum" + output = [] + for filename in filesystem.get_filenames_by_type(save_type): + json = load_file_by_type(filename, filesystem, save_type) + name = json['SaveName'] + output.append((name, filename)) + if sort_key: + output = sorted(output, key=sort_key) + return output + + +def download_file(filesystem, ident, save_type): + """Attempt to download all files for a given savefile""" + log = tts.logger() + log.info("Downloading %s file %s (from %s)" % (save_type.name, ident, filesystem)) + filename = filesystem.get_json_filename_for_type(ident, save_type) + if not filename: + log.error("Unable to find data file.") + return False + try: + data = load_json_file(filename) + except IOError as e: + log.error("Unable to read data file %s (%s)" % (filename, e)) + return False + if not data: + log.error("Unable to read data file %s" % filename) + return False + + save = tts.Save(savedata=data, + filename=filename, + ident=ident, + save_type=save_type, + filesystem=filesystem) + + if save.is_installed: + log.info("All files already downloaded.") + return True + + successful = save.download() + if successful: + log.info("All files downloaded.") + else: + log.info("Some files failed to download.") + return successful diff --git a/tts/url.py b/tts/url.py index ecfa6db..eac8098 100644 --- a/tts/url.py +++ b/tts/url.py @@ -1,107 +1,129 @@ -import urllib.request -import urllib.error import http.client -import imghdr -import tts +import os +import urllib.error +import urllib.request -# fix jpeg detection -def test_jpg(h,f): - """binary jpg""" - if h[:3]==b'\xff\xd8\xff': - return 'jpg' +import tts.util +from tts.filetype import FileType -imghdr.tests.append(test_jpg) class Url: - def __init__(self,url,filesystem): - self.url = url - self.stripped_url=tts.strip_filename(url) - self.filesystem = filesystem - self._isImage=None - self._looked_for_location=False - self._location=None + def __init__(self, url: str, type: FileType, filesystem: "FileSystem"): + self.url = url + self.stripped_url = tts.util.strip_filename(url) + self.filesystem = filesystem + self._type = type + self._looked_for_location = False + self._location = None + self._extension = None + + def examine_filesystem(self): + if not self._looked_for_location: + result = self.filesystem.check_for_file_location(self.url, self._type) + if result: + # It seems I can't return a tuple of (None, None), so... + self._location, self._extension = result + self._looked_for_location = True + + def is_unavailiable(self): + """ Check whether this url can be reached. + """ + log = tts.logger() + url = self.url + protocols = url.split('://') + if len(protocols) == 1: + log.warn(f"Missing protocol for {url}. Assuming http://.") + url = "http://" + url + log.info(f"Downloading data for {url}") + user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' + headers = {'User-Agent': user_agent} + request = urllib.request.Request(url, headers=headers) + try: + response = urllib.request.urlopen(request) + except urllib.error.URLError: + return True + # In theory it might still not be downloadable at this point as the server + # might only have a partial copy. Going to assume that is unlikely enough we can ignore it. + return response.status >= 400 - def examine_filesystem(self): - if not self._looked_for_location: - self._location,self._isImage=self.filesystem.find_details(self.url) - self._looked_for_location=True + def download(self): + log = tts.logger() + if self.exists: + return True + if self._type is FileType.NONE: + log.info("Skipping none type file") + return True + url = self.url + protocols = url.split('://') + if len(protocols) == 1: + log.warn("Missing protocol for {}. Assuming http://.".format(url)) + url = "http://" + url + log.info("Downloading data for %s." % url) + user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' + headers = {'User-Agent': user_agent} + request = urllib.request.Request(url, headers=headers) + try: + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + log.error("Error downloading %s (%s)" % (url, e)) + return False + try: + data = response.read() + except http.client.IncompleteRead as e: + # This error is the http server did not return the whole file + log.error("Error downloading %s (%s)" % (url, e)) + return False - def download(self): - log=tts.logger() - if self.exists: - return True - url=self.url - protocols=url.split('://') - if len(protocols)==1: - log.warn("Missing protocol for {}. Assuming http://.".format(url)) - url = "http://" + url - log.info("Downloading data for %s." % url) - user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' - headers = { 'User-Agent' : user_agent } - request=urllib.request.Request(url,headers=headers) - try: - response=urllib.request.urlopen(request) - except urllib.error.URLError as e: - log.error("Error downloading %s (%s)" % (url,e)) - return False - try: - data=response.read() - except http.client.IncompleteRead as e: - #This error is the http server did not return the whole file - log.error("Error downloading %s (%s)" % (url,e)) - return False - imagetype=imghdr.what('',data) - filename=None - if imagetype==None: - filename=self.filesystem.get_model_path(self.stripped_url+'.obj') - log.debug("File is OBJ") - else: - if imagetype=='jpeg': - imagetype='jpg' - log.debug("File is %s" % imagetype) - filename=self.filesystem.get_image_path(self.stripped_url+'.'+imagetype) - try: - fh=open(filename,'wb') - fh.write(data) - fh.close() - except IOError as e: - log.error("Error writing file %s (%s)" % (filename,e)) - return False - self._looked_for_location=False - return True + self._extension = self._type.get_extension(data) - @property - def exists(self): - """Does the url exist on disk already?""" - return self.location != None + filename = os.path.join( + self.filesystem.get_dir(self._type), + f"{self.stripped_url}{self._extension}" + ) + log.info(f"Writing file to {filename}") + try: + fh = open(filename, 'wb') + fh.write(data) + fh.close() + except IOError as e: + log.error("Error writing file %s (%s)" % (filename, e)) + return False + self._looked_for_location = False + return True - @property - def isImage(self): - """Do we think this is an image?""" - self.examine_filesystem() - return self._isImage + @property + def exists(self): + """Does the url exist on disk already?""" + return self.location is not None - @property - def location(self): - """Return the location of the file on disk for this url, if it exists.""" - self.examine_filesystem() - return self._location + @property + def type(self): + return self._type - def __repr__(self): - if self.exists: - return "%s: %s (%s)" % ( \ - "Image" if self.isImage else "Model", \ - self.url, \ - self.location) - else: - return "%s (Not Found)" % self.url + @property + def isImage(self): + """Do we think this is an image?""" + return self._type == FileType.IMAGE - def __str__(self): - if self.exists: - return "%s: %s" % ( \ - "Image" if self.isImage else "Model", \ - self.url) - else: - return "%s (Not Found)" % self.url + @property + def location(self): + """Return the location of the file on disk for this url, if it exists.""" + self.examine_filesystem() + return self._location + def __repr__(self): + if self.exists: + return "%s: %s (%s)" % ( + "Image" if self.isImage else "Model", + self.url, + self.location) + else: + return "%s (Not Found)" % self.url + def __str__(self): + if self.exists: + return "%s: %s" % ( + "Image" if self.isImage else "Model", + self.url) + else: + return "%s (Not Found)" % self.url diff --git a/tts/util.py b/tts/util.py new file mode 100644 index 0000000..af68d79 --- /dev/null +++ b/tts/util.py @@ -0,0 +1,27 @@ +import string +import imghdr + + +# fix jpeg detection +def test_jpg(h, f): + """binary jpg""" + if h[:3] == b'\xff\xd8\xff': + return 'jpg' + + +imghdr.tests.append(test_jpg) + + +def strip_filename(filename): + # Convert a filename to TTS format. + valid_chars = "%s%s" % (string.ascii_letters, string.digits) + return ''.join(c for c in filename if c in valid_chars) + + +def identify_image_extension(data) -> str: + image_type = imghdr.what("", data) + if image_type == 'jpeg': + image_type = 'jpg' + if image_type: + return "." + image_type + return None diff --git a/tts_cli.py b/tts_cli.py index a2cd5bc..d46ba04 100755 --- a/tts_cli.py +++ b/tts_cli.py @@ -1,258 +1,281 @@ #!/usr/bin/env python3 -import tts +import _io import argparse +import logging import os.path import sys -import codecs -import locale -import _io -import json -import zipfile -import logging + +import tts + class TTS_CLI: - def __init__(self): - self.preferences=tts.preferences.Preferences() + def __init__(self): + self.preferences = tts.preferences.Preferences() - parser = argparse.ArgumentParser(description="Manipulate Tabletop Simulator files") - parser.add_argument("-d","--directory",help="Override TTS cache directory") - parser.add_argument("-l","--loglevel",help="Set logging level",choices=['debug','info','warn','error']) - subparsers = parser.add_subparsers(dest='parser',title='command',description='Valid commands.') - subparsers.required=True + parser = argparse.ArgumentParser(description="Manipulate Tabletop Simulator files") + parser.add_argument("-d", "--directory", help="Override TTS cache directory") + parser.add_argument("-l", "--loglevel", help="Set logging level", choices=['debug', 'info', 'warn', 'error']) + subparsers = parser.add_subparsers(dest='parser', title='command', description='Valid commands.') + subparsers.required = True - # add list command - parser_list = subparsers.add_parser('list',help="List installed mods.",description=''' + # add list command + parser_list = subparsers.add_parser('list', help="List installed mods.", description=''' List installed mods. If no id is provided, then this will return a list of all installed modules. If an id is provided, then this will list the contents of that modules. ''') - group_list=parser_list.add_mutually_exclusive_group() - group_list.add_argument("-w","--workshop",action="store_const",metavar='save_type',dest='save_type',const=tts.SaveType.workshop,help="List workshop files (the default).") - group_list.add_argument("-s","--save",action="store_const",metavar='save_type',dest='save_type',const=tts.SaveType.save,help="List saves.") - group_list.add_argument("-c","--chest",action="store_const",metavar='save_type',dest='save_type',const=tts.SaveType.chest,help="List chest files.") - - parser_list.add_argument("id",nargs='?',help="ID of specific mod to list details of.") - parser_list.set_defaults(func=self.do_list) - - # export command - parser_export = subparsers.add_parser('export',help="Export a mod.",description='Export a mod in a format suitible for later import.') - group_export=parser_export.add_mutually_exclusive_group() - group_export.add_argument("-w","--workshop",action="store_const",dest='save_type',metavar='save_type',const=tts.SaveType.workshop,help="ID is of workshop file (the default).") - group_export.add_argument("-s","--save",action="store_const",dest='save_type',metavar='save_type',const=tts.SaveType.save,help="ID is of savegame file.") - group_export.add_argument("-c","--chest",action="store_const",dest='save_type',metavar='save_type',const=tts.SaveType.chest,help="ID is of chest file.") - parser_export.add_argument("id",help="ID of mod/name of savegame to export.") - parser_export.add_argument("-o","--output",help="Location/file to export to.") - parser_export.add_argument("-f","--force",action="store_true",help="Force creation of export file.") - parser_export.add_argument("-d","--download",action="store_true",help="Attempt to download missing cache files. (EXPERIMENTAL)") - parser_export.set_defaults(func=self.do_export) - - # import command - parser_import = subparsers.add_parser('import',help="Import a mod.",description="Import an previously exported mod.") - parser_import.add_argument("file",help="Mod pak file to import.") - parser_import.set_defaults(func=self.do_import) - - # download command - parser_download = subparsers.add_parser('download',help='Download mod files.',description='Attempt to download any missing files for an installed mod.') - group_download=parser_download.add_mutually_exclusive_group() - group_download.add_argument("-w","--workshop",action="store_const",dest='save_type',metavar='save_type',const=tts.SaveType.workshop,help="ID is of workshop file.") - group_download.add_argument("-s","--save",action="store_const",dest='save_type',metavar='save_type',const=tts.SaveType.save,help="ID is of savegame file.") - group_download.add_argument("-c","--chest",action="store_const",dest='save_type',metavar='save_type',const=tts.SaveType.chest,help="ID is of chest file.") - group_download_target=parser_download.add_mutually_exclusive_group(required=True) - group_download_target.add_argument("-a","--all",action="store_true",help="Download all.") - group_download_target.add_argument("id",nargs='?',help="ID of mod/name of savegame to download.") - parser_download.set_defaults(func=self.do_download) - - # cache command - parser_cache = subparsers.add_parser('cache',help='Work with the cache.') - subparsers_cache = parser_cache.add_subparsers(dest='parser_cache',title='cache_command',description='Valid sub-commands.') - subparsers_cache.required = True - parser_cache_create = subparsers_cache.add_parser('create',help='(re)create cache directory') - parser_cache_create.set_defaults(func=self.do_cache_create) - - # config command - parser_config = subparsers.add_parser('config',help='Configure tts manager.') - subparsers_config = parser_config.add_subparsers(dest='parser_config',title='config_command',description='Valid sub-commands.') - subparsers_config.required = True - parser_config_list = subparsers_config.add_parser('list',help='List configuration.') - parser_config_list.set_defaults(func=self.do_config_list) - parser_config_validate = subparsers_config.add_parser('validate',help='Validate configuration.') - parser_config_validate.set_defaults(func=self.do_config_validate) - parser_config_reset = subparsers_config.add_parser('reset',help='Reset configuration.') - parser_config_reset.set_defaults(func=self.do_config_reset) - parser_config_set = subparsers_config.add_parser('set',help='Set configuration parameters.') - parser_config_set.set_defaults(func=self.do_config_set) - parser_config_set.add_argument("-m","--mod_location",choices=['documents','gamedata'],help="Where mods are stored.") - parser_config_set.add_argument("-t","--tts_location",help="TTS Install directory") - - args = parser.parse_args() - - # set logging - if args.loglevel: - logmap={ - 'debug':logging.DEBUG, - 'info':logging.INFO, - 'warn':logging.WARN, - 'error':logging.ERROR - } - tts.logger().setLevel(logmap[args.loglevel]) - else: - tts.logger().setLevel(logging.WARN) - - # load filesystem values - if args.directory: - self.filesystem = tts.filesystem.FileSystem(os.path.abspath(args.directory)) - else: - self.filesystem = self.preferences.get_filesystem() - - if (args.parser=='list' or args.parser=='export') and not args.save_type: - # set default - args.save_type = tts.SaveType.workshop - - if (args.parser=='config' and args.parser_config=='set' and not args.mod_location and not args.tts_location): - parser_config_set.error("At least one of -m or -t is required.") - - rc,message = args.func(args) - if message: - print(message) - sys.exit(rc) - - def do_config_set(self,args): - if args.mod_location: - self.preferences.locationIsUser = args.mod_location=='documents' - if args.tts_location: - self.preferences.TTSLocation=args.mod_location - self.preferences.save() - return 0,"Preferences set" - - def do_config_reset(self,args): - self.preferences.reset() - return 0,"Preferences Reset." - - def do_config_list(self,args): - return 0,self.preferences - - def do_config_validate(self,args): - if self.preferences.validate(): - return 0,"Configuration validated OK." - else: - return 1,"Configuration failed to validate." - - def do_cache_create(self,args): - try: - self.filesystem.create_dirs() - except OSError as exception: - return 1,"OS error: {0}".format(exception) - return 0,"All directories created OK." - - def list_by_type(self,save_type): - result="" - for (name,id) in tts.describe_files_by_type(self.filesystem,save_type): - result+="\n%s (%s)" % (name,id) - return 0,result - - def list_item(self,data,filename,ident): - if not data: - self.list_installed() - return - save=tts.Save(savedata=data,ident=ident,filename=filename,filesystem=self.filesystem) - return 0,save - - def do_download(self,args): - successful=True - if not args.all: - if not args.save_type: - args.save_type=self.filesystem.get_json_filename_type(args.id) - if not args.save_type: - return 1,"Unable to determine type of id %s" % args.id - successful = tts.download_file(self.filesystem,args.id,args.save_type) - else: - if args.save_type: - for ident in self.filesystem.get_filenames_by_type(args.save_type): - if not tts.download_file(self.filesystem,ident,args.save_type): - successful=False - break - else: - for save_type in tts.SaveType: - for ident in self.filesystem.get_filenames_by_type(save_type): - if not tts.download_file(self.filesystem,ident,save_type): - successful=False - break - - if successful: - return 0, "All files downloaded." - else: - return 1, "Some files failed to download." - - def do_list(self,args): - rc=0 - result=None - - if not args.id: - rc,result=self.list_by_type(args.save_type) - else: - if not args.save_type: - args.save_type=self.filesystem.get_json_filename_type(args.id) - if not args.save_type: - return 1,"Unable to determine type of id %s" % args.id - filename=self.filesystem.get_json_filename_for_type(args.id,args.save_type) - data=tts.load_json_file(filename) - rc,result=self.list_item(data,filename,args.id) - return rc,result - - def do_export(self,args): - filename=None - if args.output: - if os.path.isdir(args.output): - filename=os.path.join(args.output,args.id+".pak") - else: - filename=args.output - else: - filename=args.id+".pak" - - data=None - json_filename=None - if not args.save_type: - args.save_type=self.filesystem.get_json_filename_type(args.id) - if not args.save_type: - return 1,"Unable to determine type of id %s" % args.id - - json_filename=self.filesystem.get_json_filename_for_type(args.id,args.save_type) - - if not json_filename: - return 1, "Unable to find filename for id %s (wrong -s/-w/-c specified?)" % args.id - data=tts.load_json_file(json_filename) - if not data: - return 1, "Unable to load data for file %s" % json_filename - - save=tts.Save(savedata=data, - filename=json_filename, - ident=args.id, - save_type=args.save_type, - filesystem=self.filesystem) - if not save.isInstalled: - if not args.download: - return 1, "Unable to find all urls required by %s. Rerun with -d to try and download them or open it within TTS.\n%s" % (args.id,save) - else: - tts.logger().info("Downloading missing files...") - successful = save.download() + group_list = parser_list.add_mutually_exclusive_group() + group_list.add_argument("-w", "--workshop", action="store_const", metavar='save_type', dest='save_type', + const=tts.SaveType.workshop, help="List workshop files (the default).") + group_list.add_argument("-s", "--save", action="store_const", metavar='save_type', dest='save_type', + const=tts.SaveType.save, help="List saves.") + group_list.add_argument("-c", "--chest", action="store_const", metavar='save_type', dest='save_type', + const=tts.SaveType.chest, help="List chest files.") + + parser_list.add_argument("id", nargs='?', help="ID of specific mod to list details of.") + parser_list.set_defaults(func=self.do_list) + + # export command + parser_export = subparsers.add_parser('export', help="Export a mod.", + description='Export a mod in a format suitible for later import.') + group_export = parser_export.add_mutually_exclusive_group() + group_export.add_argument("-w", "--workshop", action="store_const", dest='save_type', metavar='save_type', + const=tts.SaveType.workshop, help="ID is of workshop file (the default).") + group_export.add_argument("-s", "--save", action="store_const", dest='save_type', metavar='save_type', + const=tts.SaveType.save, help="ID is of savegame file.") + group_export.add_argument("-c", "--chest", action="store_const", dest='save_type', metavar='save_type', + const=tts.SaveType.chest, help="ID is of chest file.") + parser_export.add_argument("id", help="ID of mod/name of savegame to export.") + parser_export.add_argument("-m", "--missing", action="store_true", help="Only export missing files.") + parser_export.add_argument("-o", "--output", help="Location/file to export to.") + parser_export.add_argument("-f", "--force", action="store_true", help="Force creation of export file.") + parser_export.add_argument("-d", "--download", action="store_true", + help="Attempt to download missing cache files. (EXPERIMENTAL)") + parser_export.set_defaults(func=self.do_export) + + # import command + parser_import = subparsers.add_parser('import', help="Import a mod.", + description="Import an previously exported mod.") + parser_import.add_argument("file", help="Mod pak file to import.") + parser_import.set_defaults(func=self.do_import) + + # download command + parser_download = subparsers.add_parser('download', help='Download mod files.', + description='Attempt to download any missing files for an installed mod.') + group_download = parser_download.add_mutually_exclusive_group() + group_download.add_argument("-w", "--workshop", action="store_const", dest='save_type', metavar='save_type', + const=tts.SaveType.workshop, help="ID is of workshop file.") + group_download.add_argument("-s", "--save", action="store_const", dest='save_type', metavar='save_type', + const=tts.SaveType.save, help="ID is of savegame file.") + group_download.add_argument("-c", "--chest", action="store_const", dest='save_type', metavar='save_type', + const=tts.SaveType.chest, help="ID is of chest file.") + group_download_target = parser_download.add_mutually_exclusive_group(required=True) + group_download_target.add_argument("-a", "--all", action="store_true", help="Download all.") + group_download_target.add_argument("id", nargs='?', help="ID of mod/name of savegame to download.") + parser_download.set_defaults(func=self.do_download) + + # cache command + parser_cache = subparsers.add_parser('cache', help='Work with the cache.') + subparsers_cache = parser_cache.add_subparsers(dest='parser_cache', title='cache_command', + description='Valid sub-commands.') + subparsers_cache.required = True + parser_cache_create = subparsers_cache.add_parser('create', help='(re)create cache directory') + parser_cache_create.set_defaults(func=self.do_cache_create) + + # config command + parser_config = subparsers.add_parser('config', help='Configure tts manager.') + subparsers_config = parser_config.add_subparsers(dest='parser_config', title='config_command', + description='Valid sub-commands.') + subparsers_config.required = True + parser_config_list = subparsers_config.add_parser('list', help='List configuration.') + parser_config_list.set_defaults(func=self.do_config_list) + parser_config_validate = subparsers_config.add_parser('validate', help='Validate configuration.') + parser_config_validate.set_defaults(func=self.do_config_validate) + parser_config_reset = subparsers_config.add_parser('reset', help='Reset configuration.') + parser_config_reset.set_defaults(func=self.do_config_reset) + parser_config_set = subparsers_config.add_parser('set', help='Set configuration parameters.') + parser_config_set.set_defaults(func=self.do_config_set) + parser_config_set.add_argument("-m", "--mod_location", choices=['documents', 'gamedata'], + help="Where mods are stored.") + parser_config_set.add_argument("-t", "--tts_location", help="TTS Install directory") + + args = parser.parse_args() + + # set logging + if args.loglevel: + logmap = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'error': logging.ERROR + } + tts.logger().setLevel(logmap[args.loglevel]) + else: + tts.logger().setLevel(logging.WARN) + + # load filesystem values + if args.directory: + self.filesystem = tts.filesystem.FileSystem(os.path.abspath(args.directory)) + else: + self.filesystem = self.preferences.get_filesystem() + + if (args.parser == 'list' or args.parser == 'export') and not args.save_type: + # set default + args.save_type = tts.SaveType.workshop + + if ( + args.parser == 'config' and args.parser_config == 'set' and not args.mod_location and not args.tts_location): + parser_config_set.error("At least one of -m or -t is required.") + + rc, message = args.func(args) + if message: + print(message) + sys.exit(rc) + + def do_config_set(self, args): + if args.mod_location: + self.preferences.locationIsUser = args.mod_location == 'documents' + if args.tts_location: + self.preferences.TTSLocation = args.mod_location + self.preferences.save() + return 0, "Preferences set" + + def do_config_reset(self, args): + self.preferences.reset() + return 0, "Preferences Reset." + + def do_config_list(self, args): + return 0, self.preferences + + def do_config_validate(self, args): + if self.preferences.validate(): + return 0, "Configuration validated OK." + else: + return 1, "Configuration failed to validate." + + def do_cache_create(self, args): + try: + self.filesystem.create_dirs() + except OSError as exception: + return 1, "OS error: {0}".format(exception) + return 0, "All directories created OK." + + def list_by_type(self, save_type): + result = "" + for (name, id) in tts.describe_files_by_type(self.filesystem, save_type): + result += "\n%s (%s)" % (name, id) + return 0, result + + def list_item(self, data, filename, ident): + if not data: + self.list_installed() + return + save = tts.Save(savedata=data, ident=ident, filename=filename, filesystem=self.filesystem) + return 0, save + + def do_download(self, args): + successful = True + if not args.all: + if not args.save_type: + args.save_type = self.filesystem.get_json_filename_type(args.id) + if not args.save_type: + return 1, "Unable to determine type of id %s" % args.id + successful = tts.download_file(self.filesystem, args.id, args.save_type) + else: + if args.save_type: + for ident in self.filesystem.get_filenames_by_type(args.save_type): + if not tts.download_file(self.filesystem, ident, args.save_type): + successful = False + break + else: + for save_type in tts.SaveType: + for ident in self.filesystem.get_filenames_by_type(save_type): + if not tts.download_file(self.filesystem, ident, save_type): + successful = False + break + if successful: - tts.logger().info("Files downloaded successfully.") + return 0, "All files downloaded." + else: + return 1, "Some files failed to download." + + def do_list(self, args): + rc = 0 + result = None + + if not args.id: + rc, result = self.list_by_type(args.save_type) else: - return 1, "Some files failed to download" - if os.path.isfile(filename) and not args.force: - return 1,"%s already exists. Please specify another file or use '-f'" % filename - tts.logger().info("Exporting json file %s to %s" % (args.id,filename)) - save.export(filename) - # TODO: exception handling - return 0,"Exported %s to %s" % (args.id,filename) - - def do_import(self,args): - if tts.save.importPak(self.filesystem,args.file): - return 0, f"Successfully imported {args.file} into {{TODO}}" - else: - return 1, f"Error importing {args.file}" + if not args.save_type: + args.save_type = self.filesystem.get_json_filename_type(args.id) + if not args.save_type: + return 1, "Unable to determine type of id %s" % args.id + filename = self.filesystem.get_json_filename_for_type(args.id, args.save_type) + data = tts.load_json_file(filename) + rc, result = self.list_item(data, filename, args.id) + return rc, result + + def do_export(self, args): + filename = None + extension = ".part.pak" if args.missing else ".pak" + if args.output: + if os.path.isdir(args.output): + filename = os.path.join(args.output, args.id + extension) + else: + filename = args.output + else: + filename = args.id + extension + + data = None + json_filename = None + if not args.save_type: + args.save_type = self.filesystem.get_json_filename_type(args.id) + if not args.save_type: + return 1, "Unable to determine type of id %s" % args.id + + json_filename = self.filesystem.get_json_filename_for_type(args.id, args.save_type) + + if not json_filename: + return 1, "Unable to find filename for id %s (wrong -s/-w/-c specified?)" % args.id + data = tts.load_json_file(json_filename) + if not data: + return 1, "Unable to load data for file %s" % json_filename + + save = tts.Save(savedata=data, + filename=json_filename, + ident=args.id, + save_type=args.save_type, + filesystem=self.filesystem) + if not save.is_installed: + if not args.download: + return 1, "Unable to find all urls required by %s. Rerun with -d to try and download them or open it within TTS.\n%s" % ( + args.id, save) + else: + tts.logger().info("Downloading missing files...") + successful = save.download() + if successful: + tts.logger().info("Files downloaded successfully.") + else: + return 1, "Some files failed to download" + if os.path.isfile(filename) and not args.force: + return 1, "%s already exists. Please specify another file or use '-f'" % filename + tts.logger().info("Exporting json file %s to %s" % (args.id, filename)) + if args.missing: + save.export_missing(filename) + else: + save.export(filename) + # TODO: exception handling + return 0, "Exported %s to %s" % (args.id, filename) + + def do_import(self, args): + if tts.save.import_pak(self.filesystem, args.file): + return 0, f"Successfully imported {args.file}" + else: + return 1, f"Error importing {args.file}" + if __name__ == "__main__": - # fix windows' poor unicode support - sys.stdout=_io.TextIOWrapper(sys.stdout.buffer,sys.stdout.encoding,'replace',sys.stdout.newlines,sys.stdout.line_buffering) - tts_cli=TTS_CLI() + # fix windows' poor unicode support + sys.stdout = _io.TextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', sys.stdout.newlines, + sys.stdout.line_buffering) + tts_cli = TTS_CLI() diff --git a/tts_gui.py b/tts_gui.py index 6836438..62b584c 100755 --- a/tts_gui.py +++ b/tts_gui.py @@ -72,7 +72,7 @@ def file_list_has_changed(self,now): filename=filename, save_type=tts.SaveType(self.save_type.get()), filesystem=self.filesystem) - if self.save.isInstalled: + if self.save.is_installed: self.status_label.config(text="All files found.") else: self.status_label.config(text="Some cache files missing - check details on list page.") @@ -96,7 +96,7 @@ def populate_list_frame(self,frame): self.list_sb.list_command() def update_export_frame_details(self,event): - if self.export_sb.save.isInstalled: + if self.export_sb.save.is_installed: self.downloadMissingFiles.set(False) self.downloadMissingFilesCB.config(state=Tk.DISABLED) self.exportButton.config(state=Tk.NORMAL) @@ -120,11 +120,22 @@ def pickExportTarget(self): self.targetEntry.delete(0,Tk.END) self.targetEntry.insert(0,self.export_filename) + def pickPartialExportTarget(self): + exportname = filedialog.asksaveasfilename( + parent=self.root, + initialdir=os.path.join(os.path.expanduser("~"),"Downloads"), + filetypes=[('Part PAK files','*.part.pak')], + defaultextension='part.pak', + title='Choose export target') + self.export_filename = os.path.normpath(exportname) + self.targetEntry.delete(0,Tk.END) + self.targetEntry.insert(0,self.export_filename) + def pickImportTarget(self): importname = filedialog.askopenfilename( parent=self.root, initialdir=os.path.join(os.path.expanduser("~"),"Downloads"), - filetypes=[('PAK files','*.pak')], + filetypes=[('PAK files','*.pak'),('Part PAK files','*.part.pak')], defaultextension='pak', title='Choose import target') self.import_filename = os.path.normpath(importname) @@ -132,7 +143,7 @@ def pickImportTarget(self): self.importEntry.insert(0,self.import_filename) def exportPak(self): - if not self.export_sb.save.isInstalled: + if not self.export_sb.save.is_installed: successful = self.export_sb.save.download() if not successful: messagebox.showinfo("TTS Manager","Export failed (see log)") @@ -143,7 +154,7 @@ def exportPak(self): def importPak(self): self.import_filename=self.importEntry.get() - rc=tts.save.importPak(self.filesystem,self.import_filename) + rc=tts.save.import_pak(self.filesystem, self.import_filename) if rc: messagebox.showinfo("TTS Manager","Pak imported successfully.") else: @@ -175,6 +186,16 @@ def populate_export_frame(self,frame): state=Tk.DISABLED, command=self.toggleDownloadMissing) self.downloadMissingFilesCB.pack() + self.exportMissingOnly=Tk.BooleanVar() + self.exportMissingOnly.set(False) + self.exportMissingOnlyCB=ttk.Checkbutton(targetFrame, + text="Export only currently unavailiable files", + variable=self.exportMissingOnly, + offvalue=False, + onvalue=True, + state=Tk.DISABLED, + ) + self.exportMissingOnlyCB.pack() ttk.Label(targetFrame,text="Select output file").pack() self.targetEntry=ttk.Entry(targetFrame) self.targetEntry.pack(side=Tk.LEFT,expand=Tk.Y,fill=Tk.X) @@ -202,7 +223,7 @@ def populate_import_frame(self,frame): ttk.Button(importFrame,text="Import",command=self.importPak).pack() def update_download_frame_details(self,event): - if self.download_sb.save.isInstalled: + if self.download_sb.save.is_installed: self.downloadButton.config(state=Tk.DISABLED) else: self.downloadButton.config(state=Tk.NORMAL)