diff --git a/modloader/__init__.py b/modloader/__init__.py index 05110c1..e204828 100644 --- a/modloader/__init__.py +++ b/modloader/__init__.py @@ -238,9 +238,9 @@ def main(reload_mods=False): report_duplicate_labels() - if has_steam(): - steammgr = get_instance() - steammgr.CachePersonas() + # if has_steam(): + # steammgr = get_instance() + # steammgr.CachePersonas() #TODO: Change to call the new thing... # By appending the mod folder to the import path we can do something like # `import test` to import the mod named test in the mod folder. diff --git a/modloader/modconfig.py b/modloader/modconfig.py index 8607c41..44aca39 100644 --- a/modloader/modconfig.py +++ b/modloader/modconfig.py @@ -21,6 +21,7 @@ if workshop_enabled: from steam_workshop.steam_config import has_valid_signature import steam_workshop.steamhandler as steamhandler + import steamhandler_ex BRANCHES_API = "https://api.github.com/repos/AWSW-Modding/AWSW-Modtools/branches" @@ -102,8 +103,12 @@ def github_downloadable_mods(): def steam_downloadable_mods(): # A different format, # (id, mod_name, author, desc, image_url) + + # This uses GetAllItems(), Which is affected by the QueryApi crash. + # therefore, steamhandler_ex is preferred + mods = [] - for mod in sorted(steamhandler.get_instance().GetAllItems(), key=lambda mod: mod[1]): + for mod in sorted(steamhandler_ex.get_instance().GetAllItems(), key=lambda mod: mod[1]): file_id = mod[0] create_time, modify_time, signature = mod[5:8] is_valid, verified = has_valid_signature(file_id, create_time, modify_time, signature) @@ -111,7 +116,7 @@ def steam_downloadable_mods(): mods.append(list(mod[:5])) mods[-1][3] += "\n\nVerified by {}".format(verified.username.replace("", "")) else: - print "NOT VALID SIG", mod + print "NOT VALID SIG", mod[1] # Note: printing only the mod name, instead of the whi=ole thing SIGNIFICANTLY speeds up this call return mods diff --git a/modloader/steamhandler_ex.py b/modloader/steamhandler_ex.py new file mode 100644 index 0000000..4cd252c --- /dev/null +++ b/modloader/steamhandler_ex.py @@ -0,0 +1,257 @@ +import os +import shutil +import time +import errno +import cPickle +import threading +from itertools import islice +import copy + +import renpy.config + +from steam_workshop.steamhandler import SteamMgr, PyCallback, cache +from steam_workshop import steamhandler + + +class CachedSteamMgr: + """Holds a SteamMgr instance, and caches results of problematic actions (QueryApi, which gets workshop data as a whole, not parts), + So that calls to them will not fail when done repeatedly. + In QueryApi's case, This is done to bypass an existing cache which causes failures. + It is highly recommended to use this whenever one needs lists of steam mods (for example, the mod browser). + """ + + __PAGE_CACHE_DIR = os.path.join(renpy.config.gamedir, "page_cache") + + def __init__(self, steam_manager): + if not isinstance(steam_manager, SteamMgr): + raise TypeError("steam_manager must be a steam_workshop.steamhandler.SteamMgr instance!") + self._steam_manager = steam_manager + self.threads = [] # This can create threads in QueryApi, which will persist after the call returns (as is required). + # We at least keep track of them, so that the dillagent programmer can ensure they're handled properly. + return + + + def register_callback(self, type, func): + return self._steam_manager.register_callback(type, func) + + def unregister_callback(self, type, func): + return self._steam_manager.unregister_callback(type, func) + + + def is_file_stale(self, file_path): + """True if cache file given by file_path is stale.""" + # Testing if file has been updated in the last 15 minutes. If not, stale. + # Empirically, the problematic cache always becomes stale by this point. + curr_time = time.time() + cache_time = os.path.getmtime(file_path) + return (curr_time - cache_time) >= 15 * 60 + + def get_cache_filename(self, page): + """Returns the cache filename matching this page number.""" + if not isinstance(page, int): + raise TypeError("Page number must an integer!") + if page <= 0: + raise ValueError("Page number must be positive!") + return os.path.join(self.__PAGE_CACHE_DIR, "page_{:02}.pkl".format(page)) + + + def _CallQueryApi(self, page): + """Gets super's QueryApi(page), strictly by calling QueryApi. + Fills the cache as well, and keeps python thread trapped until cache has been filled.""" + print "Cache callback: Current query callbacks:", "\n".join(str(func) for func in self._steam_manager.Callbacks[PyCallback.Query]) + print "Cache callback: Current persona callbacks:", "\n".join(str(func) for func in self._steam_manager.Callbacks[PyCallback.Persona]) + + + def fill_cache_query_cb(array, arr_len): + try: + print "Cache callback called with: (len={0}), array={1}".format(arr_len, array) + + # Get cache file name + cache_file_name = self.get_cache_filename(page) + print "Cache file target: \"{}\"\n".format(cache_file_name) + + # Ensure file can be created (ensure directories) + to_ensure = os.path.dirname(cache_file_name) + if not os.path.exists(to_ensure): # If not exists: create + os.makedirs(os.path.dirname(cache_file_name)) + elif not os.path.isdir(to_ensure): # If exists and not dir: problem + raise OSError(errno.ENOTDIR, "The attempted directory \"{}\" exists and is not a directory.".format(to_ensure)) + # else: already done + + # Write cache file + with open(cache_file_name, "wb") as cache_file: + cPickle.dump(arr_len, cache_file) + cPickle.dump(tuple(islice(array, arr_len)), cache_file) + + except Exception as e: + print "Cache callback raised Exception:", str(e) + finally: + print "Cache callback done." + fill_cache_query_cb.done = True + return + + + self.register_callback(PyCallback.Query, fill_cache_query_cb) + try: + fill_cache_query_cb.done = False + + print "Calling QueryApi({})".format(page) + self._steam_manager.QueryApi(page) + + reps = 0 + while not fill_cache_query_cb.done: + print "Waiting for cache callback, rep {}".format(int(reps)) + reps += 1 + time.sleep(1) + print "Done cache callback" + return + + finally: + self.unregister_callback(PyCallback.Query, fill_cache_query_cb) + return + + def QueryApi(self, page): + """Gets super's QueryApi(page), using the cache if available and not stale. + Cache is not stale for about 15 minutes, after which the problematic one should also be stale and not fail the program.""" + + print "Called cached queryAPI with page={}".format(page) + + # Get most recent cache file if exists + cache_file_name = self.get_cache_filename(page) + + print "cache file is: \"{}\"".format(cache_file_name) + + # Check if cache file is available for use, and try to reclaim it if it is detected as a non-file entity. + is_cache_availbable = False + if os.path.exists(cache_file_name): + print "cache file Exists" + if os.path.isfile(cache_file_name): + print "cache file is a file" + is_cache_availbable = not self.is_file_stale(cache_file_name) + elif os.path.islink(cache_file_name): + print "cache file is a link to dir" + os.unlink(cache_file_name) # Clear symlink to directory in the position of the cache file... + else: + print "cache file is a dir" + shutil.rmtree(cache_file_name) # Clear directory in the position of the cache file... + + + if is_cache_availbable: + print "Using cache file" + # Read cache file + print "Reading cahce file \"{}\"".format(cache_file_name) + with open(cache_file_name, "rb") as cache_file: + arr_len = cPickle.load(cache_file) + array = cPickle.load(cache_file) + print arr_len + + # Thread is used to match the behaviour of QueryApi, where the function returns quickly and before the callbacks are called + qapi_thread = threading.Thread(target=self._steam_manager.query_callback, kwargs={"array": array, "arr_len": arr_len}) + self.threads.append(qapi_thread) + qapi_thread.start() + else: + print "Not using cache file" + self._CallQueryApi(page) + return + + + + def GetSubscribedItems(self): + return self._steam_manager.GetSubscribedItems() + + def GetAllItems(self, get_all=False): + + # Implemented here with a performance boost (see commented out print of item), + # And with fix to overzealous repeat calls + + # It seems the only way the callback can access these variables is through global variables + # Be careful! + results = [] + + def cb(array, arr_len): + print "Recieve items..." + cb.complete = False + # Querying a page is 50 results maximum + if arr_len == 51: + cb.should_run_next = False + cb.complete = True + return + + for x in range(arr_len): + item = array[x] + if get_all: + all_data = copy.deepcopy((item.m_nPublishedFileId, item.m_eResult, item.m_eFileType, + item.m_nCreatorAppID, item.m_nConsumerAppID, item.m_rgchTitle, + item.m_rgchDescription, item.m_ulSteamIDOwner, item.m_rtimeCreated, + item.m_rtimeUpdated, item.m_rtimeAddedToUserList, item.m_eVisibility, + item.m_bBanned, item.m_bAcceptedForUse, item.m_bTagsTruncated, + item.m_rgchTags, item.m_hFile, item.m_hPreviewFile, item.m_pchFileName, + item.m_nFileSize, item.m_nPreviewFileSize, item.m_rgchURL, item.m_unVotesUp, + item.m_unVotesDown, item.m_flScore, item.m_unNumChildren, + item.m_pchPreviewLink, item.m_metadata)) + results.append(all_data) + else: + not_all_data = copy.deepcopy((item.m_nPublishedFileId, item.m_rgchTitle, item.m_ulSteamIDOwner, + item.m_rgchDescription, item.m_pchPreviewLink, item.m_rtimeCreated, + item.m_rtimeUpdated, item.m_metadata)) + results.append(not_all_data) + + cb.should_run_next = (arr_len == 50) + cb.i += 1 + cb.complete = True + return + + cb.should_run_next = True + cb.i = 1 + cb.complete = False + + self.register_callback(PyCallback.Query, cb) + + while cb.should_run_next: + cb.complete = False # Important! make sure that consecutive runs don't claim that the function is already finished! + self.QueryApi(cb.i) + + # Block + while not cb.complete: + pass + + # # Remove duplicates + # results = {item[0]: item for item in results}.values() + + if not get_all: + adj_results = [] + for i, item in enumerate(results): + print "Getting persona", i, item[1] #, item # Printing full items made this ~100x slower... + item = list(item) + item[2] = self.GetPersona(item[2]) + adj_results.append(tuple(item)) + results = adj_results + + self.unregister_callback(PyCallback.Query, cb) + + return results + + + def GetItemFromID(self, id): + return self._steam_manager.GetItemFromID(id) + + @cache + def GetPersona(self, id): + return self._steam_manager.GetPersona(id) + + @cache + def GetItemDownloadInfo(self, id): + return self._steam_manager.GetItemDownloadInfo(id) + + + + + +def get_instance(): + global _cached_instance + + if "_cached_instance" not in globals(): + _cached_instance = CachedSteamMgr(steamhandler.get_instance()) + + print "steamhandler_ex id={}".format(id(_cached_instance)) + return _cached_instance \ No newline at end of file diff --git a/mods/core/download_mods.rpy b/mods/core/download_mods.rpy index bd70844..c7ef295 100644 --- a/mods/core/download_mods.rpy +++ b/mods/core/download_mods.rpy @@ -132,6 +132,16 @@ init python: init -1 python: + import math + import threading + + from modloader import modconfig + + # Cache mod validity to expedite mod browser startup + valid_modlist_thread = threading.Thread(target=modconfig.steam_downloadable_mods) + valid_modlist_thread.start() + + def _mod_check_internet_downloader(use_steam): if internet_on(): # (modid, name, author, description, image) (for github) @@ -142,14 +152,140 @@ init -1 python: from modloader.modconfig import github_downloadable_mods as download_mods contents = download_mods() - renpy.show_screen('modmenu_download', contents=contents, use_steam=use_steam) + renpy.show_screen('modmenu_paged', contents=contents, use_steam=use_steam) else: renpy.show_screen('modmenu_nointernet') -screen modmenu_download(contents, use_steam): + + # Paging methods + def _get_slice_lims_from_page(page, page_size): + return page_size * (page - 1), page_size * page # Pages are 1-indexed but lists are 0-indexed, so 1 is subtracted from page# to match them + + def _refresh_modlist_page(page, page_size, modlist, use_steam): + start, end = _get_slice_lims_from_page(page, page_size) + renpy.hide_screen('modmenu_paged_modlist') + renpy.show_screen('modmenu_paged_modlist', contents=modlist[start:end], use_steam=use_steam) + return + +# Previous implementation kept, as the scrolling may be useful in the future +# screen modmenu_download(contents, use_steam): +# modal True +# +# frame id "modmenu_download" at alpha_dissolve: +# add "image/ui/ingame_menu_bg3.png" +# +# add "image/ui/ingame_menu_bg_light.png" at ingame_menu_light +# +# #Title +# text "MOD MENU": +# size 65 +# xpos 0.5 +# ypos 0.05 +# xcenter 0.5 +# yanchor 0.5 +# font "Ardnas.otf" +# +# #Close Button +# imagebutton: +# idle "image/ui/close_idle.png" +# hover "image/ui/close_hover.png" +# action [Show("modmenu", transition=dissolve), +# Hide("modmenu_mod_content", transition=dissolve), +# Hide("modmenu_download", transition=dissolve), +# Stop("modmenu_music", fadeout=1.0), +# Play("music", "mx/menu.ogg", fadein=1.0), +# Play("audio", "se/sounds/close.ogg")] +# +# xpos 0.94 +# ypos 0.02 +# +# frame: +# background None +# yminimum 900 +# ymaximum 900 +# xmaximum 425 +# xminimum 425 +# xpos 65 +# ypos 90 +# +# #button hieght 125 +# vpgrid id "modselect_vp": +# +# yminimum 900 +# ymaximum 900 +# xmaximum 425 +# xminimum 425 +# +# cols 1 +# spacing 30 +# draggable True +# mousewheel True +# +# for modid, name, author, description, url in contents: +# $ modname = modmenu_name_cleaner(name) +# +# if len(modname) <= 21: +# #if mod is installed +# if str(modid) in modinfo.get_mod_folders(): +# $ modname = modname + "\n{size=-5}(Installed){/size}" +# #if mod is not installed +# else: +# $ modname = modname +# +# #if modname is greater than 21 characters, decrese size of font by 5 +# elif len(modname) <= 25: +# #if mod is installed +# if str(modid) in modinfo.get_mod_folders(): +# $ modname = "{size=-5}" + modname + "{/size}" + "\n{size=-5}(Installed){/size}" +# #if mod is not installed +# else: +# $ modname = "{size=-5}" + modname + "{/size}" +# +# #if modname is greater than 25 characters, decrese size of font by 10 +# else: +# #if modname is greater than 30 characters, decrese size of font by 10 and cut all text after 30 places +# if len(modname) > 30: +# $ modname = modname[:30] +# +# #if mod is installed +# if str(modid) in modinfo.get_mod_folders(): +# $ modname = "{size=-10}" + modname + "{/size}" + "\n{size=-5}(Installed){/size}" +# #if mod is not installed +# else: +# $ modname = "{size=-10}" + modname + "{/size}" +# +# +# textbutton "[modname]": +# style "modmenu_select_btn" +# +# action [Hide("modmenu_mod_content"), +# Show("modmenu_mod_content", +# modid=modid, +# name=unicode(name, "utf8"), +# author=unicode(author, "utf8"), +# description=unicode(description, "utf8"), +# url=url, +# use_steam=use_steam, +# transition=dissolve), +# Play("audio", "se/sounds/open.ogg")] +# +# +# bar value YScrollValue("modselect_vp"): +# style "modmenu_select_slider" +# #yalign 0.95 + + +screen modmenu_paged(contents, use_steam): modal True - frame id "modmenu_download" at alpha_dissolve: + $ valid_modlist_thread.join() + + default current_page = 1 + default PAGE_SIZE = 6 + $ MIN_PAGE = 1 # Do note, modpage numbers are 1-indexed + $ MAX_PAGE = int(math.ceil(len(contents) / float(PAGE_SIZE))) + + frame id "modmenu_paged" at alpha_dissolve: add "image/ui/ingame_menu_bg3.png" add "image/ui/ingame_menu_bg_light.png" at ingame_menu_light @@ -168,8 +304,9 @@ screen modmenu_download(contents, use_steam): idle "image/ui/close_idle.png" hover "image/ui/close_hover.png" action [Show("modmenu", transition=dissolve), + Hide("modmenu_paged_modlist", transition=dissolve), Hide("modmenu_mod_content", transition=dissolve), - Hide("modmenu_download", transition=dissolve), + Hide("modmenu_paged", transition=dissolve), Stop("modmenu_music", fadeout=1.0), Play("music", "mx/menu.ogg", fadein=1.0), Play("audio", "se/sounds/close.ogg")] @@ -177,80 +314,120 @@ screen modmenu_download(contents, use_steam): xpos 0.94 ypos 0.02 - frame: - background None + + hbox id "page_number_hb": + yminimum 425 + ymaximum 425 + xmaximum 900 + xminimum 900 + + ypos 913 + xcenter 960 + yanchor 0.5 + + + textbutton "-5": + xalign 0.2 + ycenter 0.5 + # Tried to bind this to shift+scroll, but it didn't work... + action [SetScreenVariable("current_page", max(current_page-5, MIN_PAGE)), + Function(_refresh_modlist_page, max(current_page-5, MIN_PAGE), PAGE_SIZE, contents, use_steam=use_steam) + ] + sensitive (current_page > 1) + + textbutton "-": + xalign 0.4 + ycenter 0.5 + keysym "mousedown_4" + action [SetScreenVariable("current_page", current_page-1), + Function(_refresh_modlist_page, current_page-1, PAGE_SIZE, contents, use_steam=use_steam) + ] + sensitive (current_page > 1) + + label "Page #[current_page]/[MAX_PAGE]": + xalign 0.5 + ycenter 0.5 + + text_size 40 + + + textbutton "+": + xalign 0.6 + ycenter 0.5 + keysym "mousedown_5" + action [SetScreenVariable("current_page", current_page+1), + Function(_refresh_modlist_page, current_page+1, PAGE_SIZE, contents, use_steam=use_steam) + ] + sensitive (current_page < MAX_PAGE) + + textbutton "+5": + xalign 0.8 + ycenter 0.5 + # Also tried to bind this to shift+scroll, but it didn't work... + action [SetScreenVariable("current_page", min(current_page+5, MAX_PAGE)), + Function(_refresh_modlist_page, min(current_page+5, MAX_PAGE), PAGE_SIZE, contents, use_steam=use_steam) + ] + sensitive (current_page < MAX_PAGE) + + on "show" action Function(_refresh_modlist_page, current_page, PAGE_SIZE, contents, use_steam=use_steam) + + + +screen modmenu_paged_modlist(contents, use_steam): + frame: + background None + yminimum 900 + ymaximum 900 + xmaximum 425 + xminimum 425 + xpos 65 + ypos 90 + + #button hieght 125 + vpgrid id "modselect_vp": + yminimum 900 ymaximum 900 xmaximum 425 xminimum 425 - xpos 65 - ypos 90 - - #button hieght 125 - vpgrid id "modselect_vp": - - yminimum 900 - ymaximum 900 - xmaximum 425 - xminimum 425 - - cols 1 - spacing 30 - draggable True - mousewheel True - for modid, name, author, description, url in contents: - $ modname = modmenu_name_cleaner(name) + cols 1 + spacing 30 - if len(modname) <= 21: - #if mod is installed - if str(modid) in modinfo.get_mod_folders(): - $ modname = modname + "\n{size=-5}(Installed){/size}" - #if mod is not installed - else: - $ modname = modname + for modid, name, author, description, url in contents: + $ modname = modmenu_name_cleaner(name) + if len(modname) > 21: #if modname is greater than 21 characters, decrese size of font by 5 - elif len(modname) <= 25: - #if mod is installed - if str(modid) in modinfo.get_mod_folders(): - $ modname = "{size=-5}" + modname + "{/size}" + "\n{size=-5}(Installed){/size}" - #if mod is not installed - else: - $ modname = "{size=-5}" + modname + "{/size}" + if len(modname) <= 25: + $ modname = "{size=-5}" + modname + "{/size}" #if modname is greater than 25 characters, decrese size of font by 10 else: #if modname is greater than 30 characters, decrese size of font by 10 and cut all text after 30 places - if len(modname) > 30: - $ modname = modname[:30] - - #if mod is installed - if str(modid) in modinfo.get_mod_folders(): - $ modname = "{size=-10}" + modname + "{/size}" + "\n{size=-5}(Installed){/size}" - #if mod is not installed - else: - $ modname = "{size=-10}" + modname + "{/size}" - - - textbutton "[modname]": - style "modmenu_select_btn" - - action [Hide("modmenu_mod_content"), - Show("modmenu_mod_content", - modid=modid, - name=unicode(name, "utf8"), - author=unicode(author, "utf8"), - description=unicode(description, "utf8"), - url=url, - use_steam=use_steam, - transition=dissolve), - Play("audio", "se/sounds/open.ogg")] +# if len(modname) > 30: + $ modname = modname[:30] + $ modname = "{size=-10}" + modname + "{/size}" + + if str(modid) in modinfo.get_mod_folders(): + $ modname = modname + "\n{size=-5}(Installed){/size}" + + + textbutton "[modname]": + style "modmenu_select_btn" + + action [Hide("modmenu_mod_content"), + Show("modmenu_mod_content", + modid=modid, + name=unicode(name, "utf8"), + author=unicode(author, "utf8"), + description=unicode(description, "utf8"), + url=url, + use_steam=use_steam, + ), + Play("audio", "se/sounds/open.ogg")] - bar value YScrollValue("modselect_vp"): - style "modmenu_select_slider" - #yalign 0.95