From 2e9af066c27ff6ef1b3308a0fab887e8a35ff135 Mon Sep 17 00:00:00 2001 From: V3n3RiX Date: Fri, 4 Nov 2022 01:04:41 +0000 Subject: backend : use pickle to store dependency lists across modules, split out download as separate module (remove lots of duplicated code) --- src/backend/__init__.py | 1 + src/backend/download.py | 45 +++++++++++++++++++++++++++ src/backend/install.py | 79 ++++++++---------------------------------------- src/backend/solvedeps.py | 9 ++++-- src/backend/upgrade.py | 79 ++++++++---------------------------------------- 5 files changed, 76 insertions(+), 137 deletions(-) create mode 100644 src/backend/download.py (limited to 'src') diff --git a/src/backend/__init__.py b/src/backend/__init__.py index 19276cc..c23ebe8 100644 --- a/src/backend/__init__.py +++ b/src/backend/__init__.py @@ -1,5 +1,6 @@ from .autoremove import * from .checkenv import * +from .download import * from .getenv import * from .getfs import * from .install import * diff --git a/src/backend/download.py b/src/backend/download.py new file mode 100644 index 0000000..c48b9d2 --- /dev/null +++ b/src/backend/download.py @@ -0,0 +1,45 @@ +#!/usr/bin/python3 + +import os +import pickle +import shutil +import wget +import sisyphus.getenv +import sisyphus.getfs + +def package(pkgname): + binhostURL = sisyphus.getenv.binhostURL() + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_pkg.pickle"), "rb")) + + for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): + print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) + wget.download(binhostURL + binary) + print("") + + if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): + shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) + else: + os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) + shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) + + if os.path.exists(binary.rstrip().split("/")[1]): + os.remove(binary.rstrip().split("/")[1]) + + +def world(): + binhostURL = sisyphus.getenv.binhostURL() + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_world.pickle"), "rb")) + + for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): + print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) + wget.download(binhostURL + binary) + print("") + + if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): + shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) + else: + os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) + shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) + + if os.path.exists(binary.rstrip().split("/")[1]): + os.remove(binary.rstrip().split("/")[1]) diff --git a/src/backend/install.py b/src/backend/install.py index 6883940..3f385f0 100644 --- a/src/backend/install.py +++ b/src/backend/install.py @@ -3,12 +3,11 @@ import atexit import io import os -import shutil +import pickle import subprocess import sys -import wget import sisyphus.checkenv -import sisyphus.getenv +import sisyphus.download import sisyphus.getfs import sisyphus.killemerge import sisyphus.solvedeps @@ -19,9 +18,8 @@ import sisyphus.update def start(pkgname): if sisyphus.checkenv.root(): sisyphus.update.start() - - binhostURL = sisyphus.getenv.binhostURL() - areBinaries,areSources,needsConfig = sisyphus.solvedeps.package(pkgname) + sisyphus.solvedeps.pkg(pkgname) + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_pkg.pickle"), "rb")) if needsConfig == 0: if len(areSources) == 0: @@ -29,20 +27,7 @@ def start(pkgname): os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that would be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n") if input("Would you like to proceed?" + " " + "[y/N]" + " ").lower().strip()[:1] == "y": - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.package(pkgname) portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) portageExec.wait() sisyphus.syncdb.localTable() @@ -62,9 +47,8 @@ def start(pkgname): def estart(pkgname): if sisyphus.checkenv.root(): sisyphus.update.start() - - binhostURL = sisyphus.getenv.binhostURL() - areBinaries,areSources,needsConfig = sisyphus.solvedeps.package(pkgname) + sisyphus.solvedeps.pkg(pkgname) + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_pkg.pickle"), "rb")) if needsConfig == 0: if len(areSources) == 0: @@ -72,20 +56,7 @@ def estart(pkgname): os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that would be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n") if input("Would you like to proceed?" + " " + "[y/N]" + " ").lower().strip()[:1] == "y": - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.package(pkgname) portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) portageExec.wait() sisyphus.syncdb.localTable() @@ -99,20 +70,7 @@ def estart(pkgname): print("\n" + "These are the binary packages that would be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n") print("\n" + "These are the source packages that would be merged, in order:" + "\n\n" + " ".join(areSources) + "\n\n" + "Total:" + " " + str(len(areSources)) + " " + "source package(s)" + "\n") if input("Would you like to proceed?" + " " + "[y/N]" + " ").lower().strip()[:1] == "y": - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.package(pkgname) portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + list(pkgname)) portageExec.wait() sisyphus.syncdb.localTable() @@ -135,25 +93,12 @@ def estart(pkgname): def startx(pkgname): - binhostURL = sisyphus.getenv.binhostURL() - areBinaries,areSources,needsConfig = sisyphus.solvedeps.package.__wrapped__(pkgname) #undecorate + sisyphus.solvedeps.pkg.__wrapped__(pkgname) #undecorate + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_pkg.pickle"), "rb")) os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that will be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n\n") - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.package(pkgname) portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n'] + pkgname, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # kill portage if the program dies or it's terminated by the user atexit.register(sisyphus.killemerge.start, portageExec) diff --git a/src/backend/solvedeps.py b/src/backend/solvedeps.py index 833f8e2..35454c2 100644 --- a/src/backend/solvedeps.py +++ b/src/backend/solvedeps.py @@ -1,11 +1,14 @@ #!/usr/bin/python3 import animation +import os +import pickle import subprocess +import sisyphus.getfs @animation.wait('resolving dependencies') -def package(pkgname): +def pkg(pkgname): areBinaries = [] areSources = [] needsConfig = int() @@ -34,7 +37,7 @@ def package(pkgname): isSource = portageOutput.split("]")[1].split("[")[0].strip(" ") areSources.append(isSource) - return areBinaries,areSources,needsConfig + pickle.dump([areBinaries,areSources,needsConfig], open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_pkg.pickle"), "wb")) @animation.wait('resolving dependencies') @@ -67,4 +70,4 @@ def world(): isSource = portageOutput.split("]")[1].split("[")[0].strip(" ") areSources.append(isSource) - return areBinaries,areSources,needsConfig + pickle.dump([areBinaries,areSources,needsConfig], open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_world.pickle"), "wb")) diff --git a/src/backend/upgrade.py b/src/backend/upgrade.py index aa920dd..5b51924 100644 --- a/src/backend/upgrade.py +++ b/src/backend/upgrade.py @@ -3,12 +3,11 @@ import atexit import io import os -import shutil +import pickle import subprocess import sys -import wget import sisyphus.checkenv -import sisyphus.getenv +import sisyphus.download import sisyphus.getfs import sisyphus.killemerge import sisyphus.solvedeps @@ -19,9 +18,8 @@ import sisyphus.update def start(): if sisyphus.checkenv.root(): sisyphus.update.start() - - binhostURL = sisyphus.getenv.binhostURL() - areBinaries,areSources,needsConfig = sisyphus.solvedeps.world() + sisyphus.solvedeps.world() + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_world.pickle"), "rb")) if needsConfig == 0: if len(areSources) == 0: @@ -29,20 +27,7 @@ def start(): os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that would be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n") if input("Would you like to proceed?" + " " + "[y/N]" + " ").lower().strip()[:1] == "y": - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.world() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) portageExec.wait() sisyphus.syncdb.localTable() @@ -62,9 +47,8 @@ def start(): def estart(): if sisyphus.checkenv.root(): sisyphus.update.start() - - binhostURL = sisyphus.getenv.binhostURL() - areBinaries,areSources,needsConfig = sisyphus.solvedeps.world() + sisyphus.solvedeps.world() + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_world.pickle"), "rb")) if needsConfig == 0: if len(areSources) == 0: @@ -72,20 +56,7 @@ def estart(): os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that would be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n") if input("Would you like to proceed?" + " " + "[y/N]" + " ").lower().strip()[:1] == "y": - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.world() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) portageExec.wait() sisyphus.syncdb.localTable() @@ -99,20 +70,7 @@ def estart(): print("\n" + "These are the binary packages that would be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n") print("\n" + "These are the source packages that would be merged, in order:" + "\n\n" + " ".join(areSources) + "\n\n" + "Total:" + " " + str(len(areSources)) + " " + "source package(s)" + "\n") if input("Would you like to proceed?" + " " + "[y/N]" + " ").lower().strip()[:1] == "y": - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.world() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world']) portageExec.wait() sisyphus.syncdb.localTable() @@ -135,8 +93,8 @@ def estart(): def startx(): - binhostURL = sisyphus.getenv.binhostURL() - areBinaries,areSources,needsConfig = sisyphus.solvedeps.world.__wrapped__() #undecorate + sisyphus.solvedeps.world.__wrapped__() #undecorate + areBinaries,areSources,needsConfig = pickle.load(open(os.path.join(sisyphus.getfs.portageMetadataDir, "sisyphus_solvedeps_world.pickle"), "rb")) if not len(areSources) == 0: print("\n" + "Source package(s) found in the mix;" + " " + "Use sisyphus CLI:" + " " + "'" + "sisyphus upgrade --ebuild" + "'" + " " + "to perform the upgrade;" + " " + "Aborting." + "\n") @@ -144,20 +102,7 @@ def startx(): if not len(areBinaries) == 0: os.chdir(sisyphus.getfs.portageCacheDir) print("\n" + "These are the binary packages that will be merged, in order:" + "\n\n" + " ".join(areBinaries) + "\n\n" + "Total:" + " " + str(len(areBinaries)) + " " + "binary package(s)" + "\n\n") - for index, binary in enumerate([package + '.tbz2' for package in areBinaries], start=1): - print(">>> Downloading binary ({}".format(index) + " " + "of" + " " + str(len(areBinaries)) + ")" + " " + binary) - wget.download(binhostURL + binary) - print("") - - if os.path.isdir(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])): - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - else: - os.makedirs(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0])) - shutil.move(binary.rstrip().split("/")[1], os.path.join(os.path.join(sisyphus.getfs.portageCacheDir, binary.rstrip().split("/")[0]), os.path.basename(binary.rstrip().split("/")[1]))) - - if os.path.exists(binary.rstrip().split("/")[1]): - os.remove(binary.rstrip().split("/")[1]) - + sisyphus.download.world() portageExec = subprocess.Popen(['emerge', '--quiet', '--verbose', '--update', '--deep', '--newuse', '--usepkg', '--usepkgonly', '--rebuilt-binaries', '--backtrack=100', '--with-bdeps=y', '--misspell-suggestion=n', '--fuzzy-search=n', '@world'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # kill portage if the program dies or it's terminated by the user atexit.register(sisyphus.killemerge.start, portageExec) -- cgit v1.2.3