From 5997105bebeb1f3ea15be1c9ee00e36daf5d534d Mon Sep 17 00:00:00 2001 From: V3n3RiX Date: Mon, 16 Jan 2023 13:02:03 +0000 Subject: new download method is more reliable, but slower...speed it up again --- src/backend/__init__.py | 1 + src/backend/download.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ src/backend/install.py | 23 +++--------- src/backend/upgrade.py | 14 +++---- 4 files changed, 110 insertions(+), 27 deletions(-) create mode 100644 src/backend/download.py diff --git a/src/backend/__init__.py b/src/backend/__init__.py index 9ea56c6..9c38585 100644 --- a/src/backend/__init__.py +++ b/src/backend/__init__.py @@ -1,5 +1,6 @@ from .autoremove import * from .checkenv import * +from .download import * from .getcolor import * from .getenv import * from .getfs import * diff --git a/src/backend/download.py b/src/backend/download.py new file mode 100644 index 0000000..fce0148 --- /dev/null +++ b/src/backend/download.py @@ -0,0 +1,99 @@ +#!/usr/bin/python3 + +import atexit +import io +import os +import subprocess +import pickle +import sisyphus.getfs +import sisyphus.killemerge + + +def pkgbinpkgonly(): + fetchList = [] + areBinaries, areSources, needsConfig = pickle.load(open(os.path.join( + sisyphus.getfs.portageMetadataDir, "sisyphus_pkgdeps.pickle"), "rb")) + + for index, binary in enumerate(['=' + package for package in areBinaries]): + fetchList.append(binary) + + portageExec = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', + '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(fetchList)) + portageExec.wait() + + +def pkgbinpkg(): + fetchList = [] + areBinaries, areSources, needsConfig = pickle.load(open(os.path.join( + sisyphus.getfs.portageMetadataDir, "sisyphus_pkgdeps.pickle"), "rb")) + + for index, binary in enumerate(['=' + package for package in areBinaries]): + fetchList.append(binary) + + portageExec = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', + '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(fetchList)) + portageExec.wait() + + +def xpkgbinpkgonly(): + fetchList = [] + areBinaries, areSources, needsConfig = pickle.load(open(os.path.join( + sisyphus.getfs.portageMetadataDir, "sisyphus_pkgdeps.pickle"), "rb")) + + for index, binary in enumerate(['=' + package for package in areBinaries]): + fetchList.append(binary) + + portageExec = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', '--rebuilt-binaries', + '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(fetchList), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # kill portage if the program dies or it's terminated by the user + atexit.register(sisyphus.killemerge.start, portageExec) + + for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): + print(portageOutput.rstrip()) + + portageExec.wait() + + +def worldbinpkgonly(): + fetchList = [] + areBinaries, areSources, needsConfig = pickle.load(open(os.path.join( + sisyphus.getfs.portageMetadataDir, "sisyphus_worlddeps.pickle"), "rb")) + + for index, binary in enumerate(['=' + package for package in areBinaries]): + fetchList.append(binary) + + portageExec = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', + '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(fetchList)) + portageExec.wait() + + +def worldbinpkg(): + fetchList = [] + areBinaries, areSources, needsConfig = pickle.load(open(os.path.join( + sisyphus.getfs.portageMetadataDir, "sisyphus_worlddeps.pickle"), "rb")) + + for index, binary in enumerate(['=' + package for package in areBinaries]): + fetchList.append(binary) + + portageExec = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', + '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(fetchList)) + portageExec.wait() + + +def xworldbinpkgonly(): + fetchList = [] + areBinaries, areSources, needsConfig = pickle.load(open(os.path.join( + sisyphus.getfs.portageMetadataDir, "sisyphus_worlddeps.pickle"), "rb")) + + for index, binary in enumerate(['=' + package for package in areBinaries]): + fetchList.append(binary) + + portageExec = subprocess.Popen(['emerge', '--nodeps', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', '--rebuilt-binaries', + '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(fetchList), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # kill portage if the program dies or it's terminated by the user + atexit.register(sisyphus.killemerge.start, portageExec) + + for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): + print(portageOutput.rstrip()) + + portageExec.wait() diff --git a/src/backend/install.py b/src/backend/install.py index b1bdb0a..078ecb6 100644 --- a/src/backend/install.py +++ b/src/backend/install.py @@ -7,6 +7,7 @@ import pickle import subprocess import sys import sisyphus.checkenv +import sisyphus.download import sisyphus.getcolor import sisyphus.getfs import sisyphus.killemerge @@ -32,9 +33,7 @@ def start(pkgname): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) - portageExec.wait() + sisyphus.download.pkgbinpkgonly() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) portageExec.wait() @@ -89,9 +88,7 @@ def estart(pkgname): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) - portageExec.wait() + sisyphus.download.pkgbinpkgonly() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) portageExec.wait() @@ -121,9 +118,7 @@ def estart(pkgname): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--getbinpkg', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) - portageExec.wait() + sisyphus.download.pkgbinpkg() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) portageExec.wait() @@ -182,15 +177,7 @@ def xstart(pkgname): os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that will be merged, in order:" + "\n\n" + ", ".join( areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n\n") - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--getbinpkg', '--getbinpkgonly', '--fetchonly', - '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + pkgname, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # kill portage if the program dies or it's terminated by the user - atexit.register(sisyphus.killemerge.start, portageExec) - - for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): - print(portageOutput.rstrip()) - - portageExec.wait() + sisyphus.download.xpkgbinpkgonly() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + pkgname, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # kill portage if the program dies or it's terminated by the user diff --git a/src/backend/upgrade.py b/src/backend/upgrade.py index f47c120..284e7f2 100644 --- a/src/backend/upgrade.py +++ b/src/backend/upgrade.py @@ -7,6 +7,7 @@ import pickle import subprocess import sys import sisyphus.checkenv +import sisyphus.download import sisyphus.getcolor import sisyphus.getfs import sisyphus.killemerge @@ -32,9 +33,7 @@ def start(): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No " + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--getbinpkg', '--getbinpkgonly', - '--fetchonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) - portageExec.wait() + sisyphus.download.worldbinpkgonly() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) portageExec.wait() @@ -89,9 +88,7 @@ def estart(): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--getbinpkg', '--getbinpkgonly', - '--fetchonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) - portageExec.wait() + sisyphus.download.worldbinpkgonly() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) portageExec.wait() @@ -121,9 +118,7 @@ def estart(): user_input = input(sisyphus.getcolor.bright_white + "Would you like to proceed?" + sisyphus.getcolor.reset + " " + "[" + sisyphus.getcolor.bright_green + "Yes" + sisyphus.getcolor.reset + "/" + sisyphus.getcolor.bright_red + "No" + sisyphus.getcolor.reset + "]" + " ") if user_input.lower() in ['yes', 'y', '']: - portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--getbinpkg', '--fetchonly', - '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) - portageExec.wait() + sisyphus.download.worldbinpkg() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) portageExec.wait() @@ -187,6 +182,7 @@ def xstart(): os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that will be merged, in order:" + "\n\n" + ", ".join( areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n\n") + sisyphus.download.xworldbinpkgonly() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--getbinpkg', '--getbinpkgonly', '--fetchonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # kill portage if the program dies or it's terminated by the user -- cgit v1.2.3