diff options
author | V3n3RiX <venerix@redcorelinux.org> | 2018-08-19 17:31:33 +0100 |
---|---|---|
committer | V3n3RiX <venerix@redcorelinux.org> | 2018-08-19 17:31:33 +0100 |
commit | bc9c39c776b27874dc1e9d4322a5326cb859975c (patch) | |
tree | 5031861bba676797e7e29dcc1ee2b858bd14b6a8 /src | |
parent | 7bc6faf4f8971c7b48d0ea765fb9df7d2f21b7d4 (diff) |
implement our fetch method in sisyphus-gui, autopep the code
Diffstat (limited to 'src')
-rw-r--r--[-rwxr-xr-x] | src/backend/libsisyphus.py | 152 | ||||
-rw-r--r--[-rwxr-xr-x] | src/frontend/gui/sisyphus-gui.py | 72 |
2 files changed, 178 insertions, 46 deletions
diff --git a/src/backend/libsisyphus.py b/src/backend/libsisyphus.py index 17a292f..30baffc 100755..100644 --- a/src/backend/libsisyphus.py +++ b/src/backend/libsisyphus.py @@ -22,12 +22,14 @@ mirrorCfg = '/etc/sisyphus/mirrors.conf' # only run as root (CLI + GUI frontend) + def checkRoot(): if not os.getuid() == 0: sys.exit("\nYou need root permissions to do this, exiting!\n") # only run in binary mode (binmode) or hybrid mode (mixedmode) (CLI + GUI frontend) + def checkSystemMode(): portageBinCfg = '/opt/redcore-build/conf/intel/portage/make.conf.amd64-binmode' portageCfgSym = '/etc/portage/make.conf' @@ -44,38 +46,47 @@ def checkSystemMode(): # get current mirror information, so we know where we download from (CLI + GUI frontend) + def getBinhostURL(): binhostURL = [] - portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) + portageExec = subprocess.Popen( + ['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): binhostURL = str(portageOutput.rstrip().split("=")[1].strip('\"')) return binhostURL + def getRemotePkgsURL(): remotePkgsURL = [] - portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) + portageExec = subprocess.Popen( + ['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): - remotePkgsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remotePackagesPre.csv') + remotePkgsURL = str(portageOutput.rstrip().split("=")[1].strip( + '\"').replace('packages', 'csv') + 'remotePackagesPre.csv') return remotePkgsURL + def getRemoteDscsURL(): remoteDscsURL = [] - portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) + portageExec = subprocess.Popen( + ['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): - remoteDscsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv') + remoteDscsURL = str(portageOutput.rstrip().split("=")[1].strip( + '\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv') return remoteDscsURL # download remote CSV's to be imported into the database (CLI + GUI frontend) + def fetchRemoteDatabase(): remotePkgsURL = getRemotePkgsURL() remoteDscsURL = getRemoteDscsURL() @@ -89,59 +100,72 @@ def fetchRemoteDatabase(): # generate local CSV's to be imported into the database (CLI + GUI frontend) + def makeLocalDatabase(): - subprocess.check_call(['/usr/share/sisyphus/helpers/make_local_csv']) # this is really hard to do in python, so we cheat with a bash helper script + # this is really hard to do in python, so we cheat with a bash helper script + subprocess.check_call(['/usr/share/sisyphus/helpers/make_local_csv']) # download and import remote CSV's into the database (CLI + GUI frontend) + def syncRemoteDatabase(): fetchRemoteDatabase() sisyphusdb = sqlite3.connect(sisyphusDB) sisyphusdb.cursor().execute('''drop table if exists remote_packages''') - sisyphusdb.cursor().execute('''create table remote_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') + sisyphusdb.cursor().execute( + '''create table remote_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') with open(remotePkgsDB) as rmtCsv: for row in csv.reader(rmtCsv): - sisyphusdb.cursor().execute("insert into remote_packages (category, name, version, slot) values (?, ?, ?, ?);", row) + sisyphusdb.cursor().execute( + "insert into remote_packages (category, name, version, slot) values (?, ?, ?, ?);", row) sisyphusdb.commit() sisyphusdb.close() sisyphusdb = sqlite3.connect(sisyphusDB) sisyphusdb.cursor().execute('''drop table if exists remote_descriptions''') - sisyphusdb.cursor().execute('''create table remote_descriptions (category TEXT,name TEXT,description TEXT)''') + sisyphusdb.cursor().execute( + '''create table remote_descriptions (category TEXT,name TEXT,description TEXT)''') with open(remoteDscsDB) as rmtCsv: for row in csv.reader(rmtCsv): - sisyphusdb.cursor().execute("insert into remote_descriptions (category, name, description) values (?, ?, ?);", row) + sisyphusdb.cursor().execute( + "insert into remote_descriptions (category, name, description) values (?, ?, ?);", row) sisyphusdb.commit() sisyphusdb.close() # generate and import local CSV's into the database (CLI + GUI frontend) + def syncLocalDatabase(): makeLocalDatabase() sisyphusdb = sqlite3.connect(sisyphusDB) sisyphusdb.cursor().execute('''drop table if exists local_packages''') - sisyphusdb.cursor().execute('''create table local_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') + sisyphusdb.cursor().execute( + '''create table local_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') with open(localPkgsDB) as lclCsv: for row in csv.reader(lclCsv): - sisyphusdb.cursor().execute("insert into local_packages (category, name, version, slot) values (?, ?, ?, ?);", row) + sisyphusdb.cursor().execute( + "insert into local_packages (category, name, version, slot) values (?, ?, ?, ?);", row) sisyphusdb.commit() sisyphusdb.close() # sync portage tree (CLI + GUI frontend) + def syncPortageTree(): subprocess.call(['emerge', '--sync', '--quiet']) # sync portage configuration files (CLI + GUI frontend) + def syncPortageCfg(): os.chdir(portageCfg) subprocess.call(['git', 'pull', '--quiet']) # check remote timestamps...if newer than local timestamps, sync everything (CLI + GUI frontend) + @animation.wait('syncing remote database') def syncAll(): checkRoot() @@ -150,12 +174,14 @@ def syncAll(): remoteDscsURL = getRemoteDscsURL() http = urllib3.PoolManager() - reqRemotePkgsTS = http.request('HEAD',remotePkgsURL) - remotePkgsTS = int(parser.parse(reqRemotePkgsTS.headers['last-modified']).strftime("%s")) + reqRemotePkgsTS = http.request('HEAD', remotePkgsURL) + remotePkgsTS = int(parser.parse( + reqRemotePkgsTS.headers['last-modified']).strftime("%s")) localPkgsTS = int(os.path.getctime(remotePkgsDB)) - reqRemoteDscsTS = http.request('HEAD',remoteDscsURL) - remoteDscsTS = int(parser.parse(reqRemoteDscsTS.headers['last-modified']).strftime("%s")) + reqRemoteDscsTS = http.request('HEAD', remoteDscsURL) + remoteDscsTS = int(parser.parse( + reqRemoteDscsTS.headers['last-modified']).strftime("%s")) localDscsTS = int(os.path.getctime(remoteDscsDB)) if remotePkgsTS < localPkgsTS: @@ -171,12 +197,14 @@ def syncAll(): # if something is installed with portage directly using emerge, sisyphus won't be aware of it # this will parse local portage database and import the changes into sisyphus database + @animation.wait('syncing local database') def startSyncSPM(): syncLocalDatabase() # sync portage tree and portage configuration files (CLI frontend) + @animation.wait('syncing portage') def startSync(): syncPortageTree() @@ -186,6 +214,7 @@ def startSync(): # if for some reason sisyphus database gets corrupted or deleted, we can still regenerate it from portage database # this will fetch remote information from mirrors, parse local portage database and regenerate sisyphus database + @animation.wait('resurrecting database') def rescueDB(): if os.path.exists(remotePkgsDB): @@ -202,10 +231,12 @@ def rescueDB(): # call portage to solve package(s) dependencies (CLI frontend) + @animation.wait('resolving dependencies') def solvePkgDeps(pkgList): pkgDeps = [] - portageExec = subprocess.Popen(['emerge', '-qgp'] + pkgList, stdout=subprocess.PIPE) + portageExec = subprocess.Popen( + ['emerge', '-qgp'] + pkgList, stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "/" in portageOutput.rstrip(): pkgDep = str(portageOutput.rstrip().split("]")[1].strip("\ ")) @@ -214,18 +245,22 @@ def solvePkgDeps(pkgList): # call portage to solve world dependencies (CLI frontend) + @animation.wait('resolving dependencies') def solveWorldDeps(): worldDeps = [] - portageExec = subprocess.Popen(['emerge', '-uDNqgp', '--backtrack=100', '--with-bdeps=y', '@world'], stdout=subprocess.PIPE) + portageExec = subprocess.Popen( + ['emerge', '-uDNqgp', '--backtrack=100', '--with-bdeps=y', '@world'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "/" in portageOutput.rstrip(): - worldDep = str(portageOutput.rstrip().split("]")[1].split("[")[0].strip("\ ")) + worldDep = str(portageOutput.rstrip().split("]")[ + 1].split("[")[0].strip("\ ")) worldDeps.append(worldDep) return worldDeps # fetch binaries and call portage to install the package(s) from local cache (CLI frontend) + def startInstall(pkgList): syncAll() @@ -247,17 +282,22 @@ def startInstall(pkgList): for index, binpkg in enumerate(pkgBins): subprocess.call(['qtbz2', '-x'] + str(binpkg + '.tbz2').split()) - CATEGORY = subprocess.check_output(['qxpak', '-x', '-O'] + str(binpkg + '.xpak').split() + ['CATEGORY']) - os.remove(str(binpkg + '.xpak')) # we extracted the categories, safe to delete + CATEGORY = subprocess.check_output( + ['qxpak', '-x', '-O'] + str(binpkg + '.xpak').split() + ['CATEGORY']) + # we extracted the categories, safe to delete + os.remove(str(binpkg + '.xpak')) if os.path.isdir(portageCache + CATEGORY.decode().strip()): - shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) + shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) else: os.makedirs(portageCache + CATEGORY.decode().strip()) - shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) + shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) if os.path.exists(str(binpkg + '.tbz2')): - os.remove(str(binpkg + '.tbz2')) # we moved the binaries in cache, safe to delete + # we moved the binaries in cache, safe to delete + os.remove(str(binpkg + '.tbz2')) portageExec = subprocess.Popen(['emerge', '-q'] + pkgList) portageExec.wait() @@ -265,6 +305,7 @@ def startInstall(pkgList): # call portage to uninstall the package(s) (CLI frontend) + def startUninstall(pkgList): portageExec = subprocess.Popen(['emerge', '-cqa'] + pkgList) portageExec.wait() @@ -272,6 +313,7 @@ def startUninstall(pkgList): # call portage to force-uninstall the package(s) (CLI frontend) + def startUninstallForce(pkgList): portageExec = subprocess.Popen(['emerge', '-Cqa'] + pkgList) portageExec.wait() @@ -279,6 +321,7 @@ def startUninstallForce(pkgList): # call portage to remove orphan package(s) (CLI frontend) + def removeOrphans(): portageExec = subprocess.Popen(['emerge', '-cqa']) portageExec.wait() @@ -286,6 +329,7 @@ def removeOrphans(): # fetch binaries and call portage to perform a system upgrade using local cache (CLI frontend) + def startUpgrade(): syncAll() @@ -307,51 +351,63 @@ def startUpgrade(): for index, worldpkg in enumerate(worldBins): subprocess.call(['qtbz2', '-x'] + str(worldpkg + '.tbz2').split()) - CATEGORY = subprocess.check_output(['qxpak', '-x', '-O'] + str(worldpkg + '.xpak').split() + ['CATEGORY']) - os.remove(str(worldpkg + '.xpak')) # we extracted the categories, safe to delete + CATEGORY = subprocess.check_output( + ['qxpak', '-x', '-O'] + str(worldpkg + '.xpak').split() + ['CATEGORY']) + # we extracted the categories, safe to delete + os.remove(str(worldpkg + '.xpak')) if os.path.isdir(portageCache + CATEGORY.decode().strip()): - shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) + shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) else: os.makedirs(portageCache + CATEGORY.decode().strip()) - shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) + shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) if os.path.exists(str(worldpkg + '.tbz2')): - os.remove(str(worldpkg + '.tbz2')) # we moved the binaries in cache, safe to delete + # we moved the binaries in cache, safe to delete + os.remove(str(worldpkg + '.tbz2')) - portageExec = subprocess.Popen(['emerge', '-uDNq', '--backtrack=100', '--with-bdeps=y', '@world']) + portageExec = subprocess.Popen( + ['emerge', '-uDNq', '--backtrack=100', '--with-bdeps=y', '@world']) portageExec.wait() syncLocalDatabase() # call portage to search for package(s) (CLI frontend) + def startSearch(pkgList): - subprocess.check_call(['emerge', '-sg'] + pkgList) # FIXME : query sisyphus.db instead of searching through portage + # FIXME : query sisyphus.db instead of searching through portage + subprocess.check_call(['emerge', '-sg'] + pkgList) # check remote timestamps...if newer than local timestamps, sync everything (CLI + GUI frontend) + def startUpdate(): syncAll() # display information about installed core packages and portage configuration (CLI frontend) + def sysInfo(): subprocess.check_call(['emerge', '--info']) # kill background portage process if sisyphus dies (CLI + GUI frontend) + def portageKill(portageCmd): - portageCmd.terminate() + portageCmd.terminate() # get a list of mirrors (GUI frontend) + def getMirrors(): mirrorList = [] with open(mirrorCfg) as mirrorFile: for line in mirrorFile.readlines(): if 'PORTAGE_BINHOST=' in line: url = line.split("=")[1].replace('"', '').rstrip() - mirror = {'isActive':True,'Url':url} + mirror = {'isActive': True, 'Url': url} if line.startswith('#'): mirror['isActive'] = False mirrorList.append(mirror) @@ -360,14 +416,19 @@ def getMirrors(): # set the active mirror (GUI frontend) + def setActiveMirror(mirrorList): with open(mirrorCfg, 'w+') as mirrorFile: - mirrorFile.write("#######################################################\n") - mirrorFile.write("# Support for multiple mirrors is somewhat incomplete #\n") - mirrorFile.write("# Uncomment only one mirror from the list bellow #\n") - mirrorFile.write("#######################################################\n") + mirrorFile.write( + "#######################################################\n") + mirrorFile.write( + "# Support for multiple mirrors is somewhat incomplete #\n") + mirrorFile.write( + "# Uncomment only one mirror from the list bellow #\n") + mirrorFile.write( + "#######################################################\n") mirrorFile.write("\n") - for line in mirrorList : + for line in mirrorList: mirror = 'PORTAGE_BINHOST=' + '"' + line['Url'] + '"' if not line['isActive']: mirror = '# ' + mirror @@ -376,25 +437,27 @@ def setActiveMirror(mirrorList): # get a list of mirrors (CLI frontend) + def listRepo(): mirrorList = getMirrors() for i, line in enumerate(mirrorList): if line['isActive']: - print(i+1,'*',line['Url']) + print(i + 1, '*', line['Url']) else: - print(i+1,' ',line['Url']) + print(i + 1, ' ', line['Url']) # set the active mirror (CLI frontend) + def setRepo(mirror): mirror = int(mirror[0]) mirrorList = getMirrors() - if mirror not in range(1,len(mirrorList)+1): + if mirror not in range(1, len(mirrorList) + 1): print('mirror index is wrong, please check with "sisyphus mirror list"') else: - for i in range(0,len(mirrorList)): - indx = i+1 - if indx == mirror : + for i in range(0, len(mirrorList)): + indx = i + 1 + if indx == mirror: mirrorList[i]['isActive'] = True else: mirrorList[i]['isActive'] = False @@ -402,6 +465,7 @@ def setRepo(mirror): # display help menu (CLI frontend) + def showHelp(): print("\nUsage : sisyphus command [package(s)] || [file(s)]\n") print("Sisyphus is a simple python wrapper around portage, gentoolkit, and portage-utils that provides") diff --git a/src/frontend/gui/sisyphus-gui.py b/src/frontend/gui/sisyphus-gui.py index 0f290c8..793a30e 100755..100644 --- a/src/frontend/gui/sisyphus-gui.py +++ b/src/frontend/gui/sisyphus-gui.py @@ -4,6 +4,8 @@ import subprocess import sqlite3 import io import atexit +import wget +import shutil from collections import OrderedDict from PyQt5 import QtCore, QtGui, QtWidgets, uic from libsisyphus import * @@ -372,6 +374,39 @@ class MainWorker(QtCore.QObject): def startInstall(self): self.started.emit() pkgList = Sisyphus.pkgList + + binhostURL = getBinhostURL() + pkgDeps = solvePkgDeps(pkgList) + pkgBins = [] + + for index, url in enumerate([binhostURL + package + '.tbz2' for package in pkgDeps]): + self.strReady.emit(">>> Fetching" + " " + url) + wget.download(url) + print("\n") + + for index, binpkg in enumerate(pkgDeps): + pkgBin = str(binpkg.rstrip().split("/")[1]) + pkgBins.append(pkgBin) + + for index, binpkg in enumerate(pkgBins): + subprocess.call(['qtbz2', '-x'] + str(binpkg + '.tbz2').split()) + CATEGORY = subprocess.check_output( + ['qxpak', '-x', '-O'] + str(binpkg + '.xpak').split() + ['CATEGORY']) + # we extracted the categories, safe to delete + os.remove(str(binpkg + '.xpak')) + + if os.path.isdir(portageCache + CATEGORY.decode().strip()): + shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) + else: + os.makedirs(portageCache + CATEGORY.decode().strip()) + shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) + + if os.path.exists(str(binpkg + '.tbz2')): + # we moved the binaries in cache, safe to delete + os.remove(str(binpkg + '.tbz2')) + portageExec = subprocess.Popen( ['emerge', '-q'] + pkgList, stdout=subprocess.PIPE) atexit.register(portageKill, portageExec) @@ -386,7 +421,7 @@ class MainWorker(QtCore.QObject): self.started.emit() pkgList = Sisyphus.pkgList portageExec = subprocess.Popen( - ['emerge', '--depclean', '-q'] + pkgList, stdout=subprocess.PIPE) + ['emerge', '-cq'] + pkgList, stdout=subprocess.PIPE) atexit.register(portageKill, portageExec) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): self.strReady.emit(portageOutput.rstrip()) @@ -397,6 +432,39 @@ class MainWorker(QtCore.QObject): @QtCore.pyqtSlot() def startUpgrade(self): self.started.emit() + + binhostURL = getBinhostURL() + worldDeps = solveWorldDeps() + worldBins = [] + + for index, url in enumerate([binhostURL + package + '.tbz2' for package in worldDeps]): + self.strReady.emit(">>> Fetching" + " " + url) + wget.download(url) + print("\n") + + for index, worldpkg in enumerate(worldDeps): + worldBin = str(worldpkg.rstrip().split("/")[1]) + worldBins.append(worldBin) + + for index, worldpkg in enumerate(worldBins): + subprocess.call(['qtbz2', '-x'] + str(worldpkg + '.tbz2').split()) + CATEGORY = subprocess.check_output( + ['qxpak', '-x', '-O'] + str(worldpkg + '.xpak').split() + ['CATEGORY']) + # we extracted the categories, safe to delete + os.remove(str(worldpkg + '.xpak')) + + if os.path.isdir(portageCache + CATEGORY.decode().strip()): + shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) + else: + os.makedirs(portageCache + CATEGORY.decode().strip()) + shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) + + if os.path.exists(str(worldpkg + '.tbz2')): + # we moved the binaries in cache, safe to delete + os.remove(str(worldpkg + '.tbz2')) + portageExec = subprocess.Popen( ['emerge', '-uDNq', '--backtrack=100', '--with-bdeps=y', '@world'], stdout=subprocess.PIPE) atexit.register(portageKill, portageExec) @@ -410,7 +478,7 @@ class MainWorker(QtCore.QObject): def cleanOrphans(self): self.started.emit() portageExec = subprocess.Popen( - ['emerge', '--depclean', '-q'], stdout=subprocess.PIPE) + ['emerge', '-cq'], stdout=subprocess.PIPE) atexit.register(portageKill, portageExec) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): self.strReady.emit(portageOutput.rstrip()) |