From 5a3414863014bec7f901cb04b9afd49834e181f4 Mon Sep 17 00:00:00 2001 From: V3n3RiX Date: Fri, 7 Apr 2023 17:47:46 +0100 Subject: backend : rewrite download in a more efficient way --- src/backend/download.py | 122 ++++++++++++++---------------------------------- src/backend/install.py | 11 +++-- src/backend/upgrade.py | 11 +++-- 3 files changed, 50 insertions(+), 94 deletions(-) diff --git a/src/backend/download.py b/src/backend/download.py index 32e27a6..7e2d4ec 100644 --- a/src/backend/download.py +++ b/src/backend/download.py @@ -4,96 +4,46 @@ import atexit import io import os import subprocess +import sys import pickle import sisyphus.getfs import sisyphus.killemerge -def dl_pbin_only(): +def start(redir_out=False, dl_world=False): dl_list = [] - bin_list, src_list, need_cfg = pickle.load( - open(os.path.join(sisyphus.getfs.p_mtd_dir, "sisyphus_pkgdeps.pickle"), "rb")) - for index, pbin in enumerate(['=' + package for package in bin_list]): - dl_list.append(pbin) - - p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list)) - p_exe.wait() - - -def dl_pbin(): - dl_list = [] - bin_list, src_list, need_cfg = pickle.load( - open(os.path.join(sisyphus.getfs.p_mtd_dir, "sisyphus_pkgdeps.pickle"), "rb")) - - for index, pbin in enumerate(['=' + package for package in bin_list]): - dl_list.append(pbin) - - p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list)) - p_exe.wait() - - -def xdl_pbin_only(): - dl_list = [] - bin_list, src_list, need_cfg = pickle.load( - open(os.path.join(sisyphus.getfs.p_mtd_dir, "sisyphus_pkgdeps.pickle"), "rb")) - - for index, pbin in enumerate(['=' + package for package in bin_list]): - dl_list.append(pbin) - - p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', '--rebuilt-binaries', - '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # kill portage if the program dies or it's terminated by the user - atexit.register(sisyphus.killemerge.start, p_exe) - - for p_out in io.TextIOWrapper(p_exe.stdout, encoding="utf-8"): - print(p_out.rstrip()) - - p_exe.wait() - - -def dl_wpbin_only(): - dl_list = [] - bin_list, src_list, need_cfg = pickle.load( - open(os.path.join(sisyphus.getfs.p_mtd_dir, "sisyphus_worlddeps.pickle"), "rb")) - - for index, pbin in enumerate(['=' + package for package in bin_list]): - dl_list.append(pbin) - - p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list)) - p_exe.wait() - - -def dl_wpbin(): - dl_list = [] - bin_list, src_list, need_cfg = pickle.load( - open(os.path.join(sisyphus.getfs.p_mtd_dir, "sisyphus_worlddeps.pickle"), "rb")) - - for index, pbin in enumerate(['=' + package for package in bin_list]): - dl_list.append(pbin) - - p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list)) - p_exe.wait() - - -def xdl_wpbin_only(): - dl_list = [] - bin_list, src_list, need_cfg = pickle.load( - open(os.path.join(sisyphus.getfs.p_mtd_dir, "sisyphus_worlddeps.pickle"), "rb")) - - for index, pbin in enumerate(['=' + package for package in bin_list]): - dl_list.append(pbin) - - p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', '--rebuilt-binaries', - '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # kill portage if the program dies or it's terminated by the user - atexit.register(sisyphus.killemerge.start, p_exe) - - for p_out in io.TextIOWrapper(p_exe.stdout, encoding="utf-8"): - print(p_out.rstrip()) - - p_exe.wait() + if dl_world: + file_path = os.path.join( + sisyphus.getfs.p_mtd_dir, "sisyphus_worlddeps.pickle") + else: + file_path = os.path.join( + sisyphus.getfs.p_mtd_dir, "sisyphus_pkgdeps.pickle") + + with open(file_path, "rb") as f: + bin_list, src_list, need_cfg = pickle.load(f) + + dl_list = [f'={package}' for package in bin_list] + + if redir_out: + p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', '--rebuilt-binaries', + '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # kill portage if the program dies or it's terminated by the user + atexit.register(sisyphus.killemerge.start, p_exe) + + for p_out in io.TextIOWrapper(p_exe.stdout, encoding="utf-8"): + print(p_out.rstrip()) + + p_exe.wait() + else: + p_exe = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', + '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(dl_list)) + try: + p_exe.wait() + except KeyboardInterrupt: + p_exe.terminate() + try: + p_exe.wait(1) + except subprocess.TimeoutExpired: + p_exe.kill() + sys.exit() diff --git a/src/backend/install.py b/src/backend/install.py index d214ebd..1316c22 100644 --- a/src/backend/install.py +++ b/src/backend/install.py @@ -33,7 +33,8 @@ def start(pkgname): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - sisyphus.download.dl_pbin_only() + sisyphus.download.start( + redir_out=False, dl_world=False) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) p_exe.wait() @@ -86,7 +87,8 @@ def estart(pkgname): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - sisyphus.download.dl_pbin_only() + sisyphus.download.start( + redir_out=False, dl_world=False) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) p_exe.wait() @@ -114,7 +116,8 @@ def estart(pkgname): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - sisyphus.download.dl_pbin() + sisyphus.download.start( + redir_out=False, dl_world=False) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) p_exe.wait() @@ -169,7 +172,7 @@ def xstart(pkgname): os.chdir(sisyphus.getfs.p_cch_dir) print("\n" + "These are the binary packages that will be merged, in order:" + "\n\n" + ", ".join(bin_list) + "\n\n" + "Total:" + " " + str(len(bin_list)) + " " + "binary package(s)" + "\n\n") - sisyphus.download.xdl_pbin_only() + sisyphus.download.start(redir_out=True, dl_world=False) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + pkgname, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # kill portage if the program dies or it's terminated by the user diff --git a/src/backend/upgrade.py b/src/backend/upgrade.py index c138533..a897078 100644 --- a/src/backend/upgrade.py +++ b/src/backend/upgrade.py @@ -33,7 +33,8 @@ def start(): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No " + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - sisyphus.download.dl_wpbin_only() + sisyphus.download.start( + redir_out=False, dl_world=True) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) p_exe.wait() @@ -86,7 +87,8 @@ def estart(): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - sisyphus.download.dl_wpbin_only() + sisyphus.download.start( + redir_out=False, dl_world=True) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) p_exe.wait() @@ -114,7 +116,8 @@ def estart(): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - sisyphus.download.dl_wpbin() + sisyphus.download.start( + redir_out=False, dl_world=True) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) p_exe.wait() @@ -174,7 +177,7 @@ def xstart(): os.chdir(sisyphus.getfs.p_cch_dir) print("\n" + "These are the binary packages that will be merged, in order:" + "\n\n" + ", ".join( bin_list) + "\n\n" + "Total:" + " " + str(len(bin_list)) + " " + "binary package(s)" + "\n\n") - sisyphus.download.xdl_wpbin_only() + sisyphus.download.start(redir_out=True, dl_world=True) p_exe = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # kill portage if the program dies or it's terminated by the user -- cgit v1.2.3