From ac10326fcf926f11fd21f69efd896b5ead503b26 Mon Sep 17 00:00:00 2001 From: V3n3RiX Date: Sat, 15 Sep 2018 20:02:44 +0100 Subject: reduce the number of database connections to speed things a bit, arrange the code a bit --- src/backend/libsisyphus.py | 307 ++++++++++++++------------------------------- 1 file changed, 92 insertions(+), 215 deletions(-) (limited to 'src/backend') diff --git a/src/backend/libsisyphus.py b/src/backend/libsisyphus.py index 55e185a..c218972 100644 --- a/src/backend/libsisyphus.py +++ b/src/backend/libsisyphus.py @@ -20,16 +20,10 @@ localPkgsDB = '/var/lib/sisyphus/csv/localPackagesPre.csv' sisyphusDB = '/var/lib/sisyphus/db/sisyphus.db' mirrorCfg = '/etc/sisyphus/mirrors.conf' -# only run as root (CLI + GUI frontend) - - def checkRoot(): if not os.getuid() == 0: sys.exit("\nYou need root permissions to do this, exiting!\n") -# only run in binary mode (binmode) or hybrid mode (mixedmode) (CLI + GUI frontend) - - def checkSystemMode(): portageBinCfg = '/opt/redcore-build/conf/intel/portage/make.conf.amd64-binmode' portageCfgSym = '/etc/portage/make.conf' @@ -44,48 +38,70 @@ def checkSystemMode(): print("\nThe system is not set to binmode, refusing to run!\n") sys.exit(1) -# get current mirror information, so we know where we download from (CLI + GUI frontend) +def getMirrorList(): + mirrorList = [] + with open(mirrorCfg) as mirrorFile: + for line in mirrorFile.readlines(): + if 'PORTAGE_BINHOST=' in line: + url = line.split("=")[1].replace('"', '').rstrip() + mirror = {'isActive': True, 'Url': url} + if line.startswith('#'): + mirror['isActive'] = False + mirrorList.append(mirror) + mirrorFile.close() + return mirrorList def getBinhostURL(): binhostURL = [] + portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) - portageExec = subprocess.Popen( - ['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): binhostURL = str(portageOutput.rstrip().split("=")[1].strip('\"')) - return binhostURL - def getRemotePkgsURL(): remotePkgsURL = [] + portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) - portageExec = subprocess.Popen( - ['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): - remotePkgsURL = str(portageOutput.rstrip().split("=")[1].strip( - '\"').replace('packages', 'csv') + 'remotePackagesPre.csv') - + remotePkgsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remotePackagesPre.csv') return remotePkgsURL - def getRemoteDscsURL(): remoteDscsURL = [] + portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) - portageExec = subprocess.Popen( - ['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): - remoteDscsURL = str(portageOutput.rstrip().split("=")[1].strip( - '\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv') - + remoteDscsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv') return remoteDscsURL -# download remote CSV's to be imported into the database (CLI + GUI frontend) +@animation.wait('resolving dependencies') +def getPkgDeps(pkgList): + pkgDeps = [] + portageExec = subprocess.Popen(['emerge', '-qgp'] + pkgList, stdout=subprocess.PIPE) + for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): + if "/" in portageOutput.rstrip(): + pkgDep = str(portageOutput.rstrip().split("]")[1].strip("\ ")) + if not "blocking" in pkgDep: + pkgDeps.append(pkgDep) + return pkgDeps + +@animation.wait('resolving dependencies') +def getWorldDeps(): + worldDeps = [] + portageExec = subprocess.Popen(['emerge', '-uDNqgp', '--backtrack=100', '--with-bdeps=y', '@world'], stdout=subprocess.PIPE) + + for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): + if "/" in portageOutput.rstrip(): + worldDep = str(portageOutput.rstrip().split("]")[1].split("[")[0].strip("\ ")) + if not "blocking" in worldDep: + worldDeps.append(worldDep) + return worldDeps def fetchRemoteDatabase(): remotePkgsURL = getRemotePkgsURL() @@ -98,74 +114,50 @@ def fetchRemoteDatabase(): with http.request('GET', remoteDscsURL, preload_content=False) as tmp_buffer, open(remoteDscsDB, 'wb') as output_file: shutil.copyfileobj(tmp_buffer, output_file) -# generate local CSV's to be imported into the database (CLI + GUI frontend) - - def makeLocalDatabase(): - # this is really hard to do in python, so we cheat with a bash helper script subprocess.check_call(['/usr/share/sisyphus/helpers/make_local_csv']) -# download and import remote CSV's into the database (CLI + GUI frontend) - - def syncRemoteDatabase(): fetchRemoteDatabase() sisyphusdb = sqlite3.connect(sisyphusDB) sisyphusdb.cursor().execute('''drop table if exists remote_packages''') - sisyphusdb.cursor().execute( - '''create table remote_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') + sisyphusdb.cursor().execute('''drop table if exists remote_descriptions''') + sisyphusdb.cursor().execute('''create table remote_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') + sisyphusdb.cursor().execute('''create table remote_descriptions (category TEXT,name TEXT,description TEXT)''') + with open(remotePkgsDB) as rmtCsv: for row in csv.reader(rmtCsv): - sisyphusdb.cursor().execute( - "insert into remote_packages (category, name, version, slot) values (?, ?, ?, ?);", row) - sisyphusdb.commit() - sisyphusdb.close() + sisyphusdb.cursor().execute("insert into remote_packages (category, name, version, slot) values (?, ?, ?, ?);", row) - sisyphusdb = sqlite3.connect(sisyphusDB) - sisyphusdb.cursor().execute('''drop table if exists remote_descriptions''') - sisyphusdb.cursor().execute( - '''create table remote_descriptions (category TEXT,name TEXT,description TEXT)''') with open(remoteDscsDB) as rmtCsv: for row in csv.reader(rmtCsv): - sisyphusdb.cursor().execute( - "insert into remote_descriptions (category, name, description) values (?, ?, ?);", row) + sisyphusdb.cursor().execute("insert into remote_descriptions (category, name, description) values (?, ?, ?);", row) + sisyphusdb.commit() sisyphusdb.close() -# generate and import local CSV's into the database (CLI + GUI frontend) - - def syncLocalDatabase(): makeLocalDatabase() sisyphusdb = sqlite3.connect(sisyphusDB) sisyphusdb.cursor().execute('''drop table if exists local_packages''') - sisyphusdb.cursor().execute( - '''create table local_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') + sisyphusdb.cursor().execute('''create table local_packages (category TEXT,name TEXT,version TEXT,slot TEXT)''') + with open(localPkgsDB) as lclCsv: for row in csv.reader(lclCsv): - sisyphusdb.cursor().execute( - "insert into local_packages (category, name, version, slot) values (?, ?, ?, ?);", row) + sisyphusdb.cursor().execute("insert into local_packages (category, name, version, slot) values (?, ?, ?, ?);", row) + sisyphusdb.commit() sisyphusdb.close() -# sync portage tree (CLI + GUI frontend) - - def syncPortageTree(): subprocess.call(['emerge', '--sync', '--quiet']) -# sync portage configuration files (CLI + GUI frontend) - - def syncPortageCfg(): os.chdir(portageCfg) subprocess.call(['git', 'pull', '--quiet']) -# check remote timestamps...if newer than local timestamps, sync everything (CLI + GUI frontend) - - @animation.wait('syncing remote database') def syncAll(): checkRoot() @@ -175,13 +167,11 @@ def syncAll(): http = urllib3.PoolManager() reqRemotePkgsTS = http.request('HEAD', remotePkgsURL) - remotePkgsTS = int(parser.parse( - reqRemotePkgsTS.headers['last-modified']).strftime("%s")) + remotePkgsTS = int(parser.parse(reqRemotePkgsTS.headers['last-modified']).strftime("%s")) localPkgsTS = int(os.path.getctime(remotePkgsDB)) reqRemoteDscsTS = http.request('HEAD', remoteDscsURL) - remoteDscsTS = int(parser.parse( - reqRemoteDscsTS.headers['last-modified']).strftime("%s")) + remoteDscsTS = int(parser.parse(reqRemoteDscsTS.headers['last-modified']).strftime("%s")) localDscsTS = int(os.path.getctime(remoteDscsDB)) if remotePkgsTS < localPkgsTS: @@ -193,28 +183,15 @@ def syncAll(): syncPortageCfg() syncRemoteDatabase() -# regenerate local CSV's and import them into the database (CLI frontend) -# if something is installed with portage directly using emerge, sisyphus won't be aware of it -# this will parse local portage database and import the changes into sisyphus database - - @animation.wait('syncing local database') def startSyncSPM(): syncLocalDatabase() -# sync portage tree and portage configuration files (CLI frontend) - - @animation.wait('syncing portage') def startSync(): syncPortageTree() syncPortageCfg() -# regenerate sisyphus database (CLI frontend) -# if for some reason sisyphus database gets corrupted or deleted, we can still regenerate it from portage database -# this will fetch remote information from mirrors, parse local portage database and regenerate sisyphus database - - @animation.wait('resurrecting database') def rescueDB(): if os.path.exists(remotePkgsDB): @@ -229,45 +206,17 @@ def rescueDB(): syncRemoteDatabase() syncLocalDatabase() -# call portage to solve package(s) dependencies (CLI frontend) - - -@animation.wait('resolving dependencies') -def solvePkgDeps(pkgList): - pkgDeps = [] - portageExec = subprocess.Popen( - ['emerge', '-qgp'] + pkgList, stdout=subprocess.PIPE) - for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): - if "/" in portageOutput.rstrip(): - pkgDep = str(portageOutput.rstrip().split("]")[1].strip("\ ")) - if not "blocking" in pkgDep: - pkgDeps.append(pkgDep) - return pkgDeps - -# call portage to solve world dependencies (CLI frontend) - - -@animation.wait('resolving dependencies') -def solveWorldDeps(): - worldDeps = [] - portageExec = subprocess.Popen( - ['emerge', '-uDNqgp', '--backtrack=100', '--with-bdeps=y', '@world'], stdout=subprocess.PIPE) - for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): - if "/" in portageOutput.rstrip(): - worldDep = str(portageOutput.rstrip().split("]")[ - 1].split("[")[0].strip("\ ")) - if not "blocking" in worldDep: - worldDeps.append(worldDep) - return worldDeps - -# fetch binaries and call portage to install the package(s) from local cache (CLI frontend) +def startSearch(pkgList): + subprocess.check_call(['emerge', '-sg'] + pkgList) +def startUpdate(): + syncAll() def startInstall(pkgList): syncAll() binhostURL = getBinhostURL() - pkgDeps = solvePkgDeps(pkgList) + pkgDeps = getPkgDeps(pkgList) pkgBins = [] if not len(pkgDeps) == 0: @@ -285,21 +234,16 @@ def startInstall(pkgList): for index, binpkg in enumerate(pkgBins): subprocess.call(['qtbz2', '-x'] + str(binpkg + '.tbz2').split()) - CATEGORY = subprocess.check_output( - ['qxpak', '-x', '-O'] + str(binpkg + '.xpak').split() + ['CATEGORY']) - # we extracted the categories, safe to delete + CATEGORY = subprocess.check_output(['qxpak', '-x', '-O'] + str(binpkg + '.xpak').split() + ['CATEGORY']) os.remove(str(binpkg + '.xpak')) if os.path.isdir(portageCache + CATEGORY.decode().strip()): - shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + - CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) + shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) else: os.makedirs(portageCache + CATEGORY.decode().strip()) - shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + - CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) + shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2')))) if os.path.exists(str(binpkg + '.tbz2')): - # we moved the binaries in cache, safe to delete os.remove(str(binpkg + '.tbz2')) portageExec = subprocess.Popen(['emerge', '-q'] + pkgList) @@ -308,38 +252,11 @@ def startInstall(pkgList): else: sys.exit(1) -# call portage to uninstall the package(s) (CLI frontend) - - -def startUninstall(pkgList): - portageExec = subprocess.Popen(['emerge', '-cqa'] + pkgList) - portageExec.wait() - syncLocalDatabase() - -# call portage to force-uninstall the package(s) (CLI frontend) - - -def startUninstallForce(pkgList): - portageExec = subprocess.Popen(['emerge', '-Cqa'] + pkgList) - portageExec.wait() - syncLocalDatabase() - -# call portage to remove orphan package(s) (CLI frontend) - - -def removeOrphans(): - portageExec = subprocess.Popen(['emerge', '-cqa']) - portageExec.wait() - syncLocalDatabase() - -# fetch binaries and call portage to perform a system upgrade using local cache (CLI frontend) - - def startUpgrade(): syncAll() binhostURL = getBinhostURL() - worldDeps = solveWorldDeps() + worldDeps = getWorldDeps() worldBins = [] if not len(worldDeps) == 0: @@ -357,84 +274,61 @@ def startUpgrade(): for index, worldpkg in enumerate(worldBins): subprocess.call(['qtbz2', '-x'] + str(worldpkg + '.tbz2').split()) - CATEGORY = subprocess.check_output( - ['qxpak', '-x', '-O'] + str(worldpkg + '.xpak').split() + ['CATEGORY']) - # we extracted the categories, safe to delete + CATEGORY = subprocess.check_output(['qxpak', '-x', '-O'] + str(worldpkg + '.xpak').split() + ['CATEGORY']) os.remove(str(worldpkg + '.xpak')) if os.path.isdir(portageCache + CATEGORY.decode().strip()): - shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + - CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) + shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) else: os.makedirs(portageCache + CATEGORY.decode().strip()) - shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + - CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) + shutil.move(str(worldpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(worldpkg + '.tbz2')))) if os.path.exists(str(worldpkg + '.tbz2')): - # we moved the binaries in cache, safe to delete os.remove(str(worldpkg + '.tbz2')) - portageExec = subprocess.Popen( - ['emerge', '-uDNq', '--backtrack=100', '--with-bdeps=y', '@world']) + portageExec = subprocess.Popen(['emerge', '-uDNq', '--backtrack=100', '--with-bdeps=y', '@world']) portageExec.wait() syncLocalDatabase() else: sys.exit("\n" + "Nothing to upgrade; quitting." + "\n") -# call portage to search for package(s) (CLI frontend) - - -def startSearch(pkgList): - # FIXME : query sisyphus.db instead of searching through portage - subprocess.check_call(['emerge', '-sg'] + pkgList) - -# check remote timestamps...if newer than local timestamps, sync everything (CLI + GUI frontend) - - -def startUpdate(): - syncAll() +def startUninstall(pkgList): + portageExec = subprocess.Popen(['emerge', '-cqa'] + pkgList) + portageExec.wait() + syncLocalDatabase() -# display information about installed core packages and portage configuration (CLI frontend) +def startUninstallForce(pkgList): + portageExec = subprocess.Popen(['emerge', '-Cqa'] + pkgList) + portageExec.wait() + syncLocalDatabase() +def removeOrphans(): + portageExec = subprocess.Popen(['emerge', '-cqa']) + portageExec.wait() + syncLocalDatabase() def sysInfo(): subprocess.check_call(['emerge', '--info']) -# kill background portage process if sisyphus dies (CLI + GUI frontend) - - def portageKill(portageCmd): portageCmd.terminate() -# get a list of mirrors (GUI frontend) - - -def getMirrors(): - mirrorList = [] - with open(mirrorCfg) as mirrorFile: - for line in mirrorFile.readlines(): - if 'PORTAGE_BINHOST=' in line: - url = line.split("=")[1].replace('"', '').rstrip() - mirror = {'isActive': True, 'Url': url} - if line.startswith('#'): - mirror['isActive'] = False - mirrorList.append(mirror) - mirrorFile.close() - return mirrorList -# set the active mirror (GUI frontend) +def printMirrorList(): + mirrorList = getMirrorList() + for i, line in enumerate(mirrorList): + if line['isActive']: + print(i + 1, '*', line['Url']) + else: + print(i + 1, ' ', line['Url']) -def setActiveMirror(mirrorList): +def writeMirrorCfg(mirrorList): with open(mirrorCfg, 'w+') as mirrorFile: - mirrorFile.write( - "#######################################################\n") - mirrorFile.write( - "# Support for multiple mirrors is somewhat incomplete #\n") - mirrorFile.write( - "# Uncomment only one mirror from the list bellow #\n") - mirrorFile.write( - "#######################################################\n") + mirrorFile.write("#######################################################\n") + mirrorFile.write("# Support for multiple mirrors is somewhat incomplete #\n") + mirrorFile.write("# Uncomment only one mirror from the list bellow #\n") + mirrorFile.write("#######################################################\n") mirrorFile.write("\n") for line in mirrorList: mirror = 'PORTAGE_BINHOST=' + '"' + line['Url'] + '"' @@ -443,23 +337,9 @@ def setActiveMirror(mirrorList): mirrorFile.write(mirror + "\n") mirrorFile.write("\n") -# get a list of mirrors (CLI frontend) - - -def listRepo(): - mirrorList = getMirrors() - for i, line in enumerate(mirrorList): - if line['isActive']: - print(i + 1, '*', line['Url']) - else: - print(i + 1, ' ', line['Url']) - -# set the active mirror (CLI frontend) - - -def setRepo(mirror): +def setActiveMirror(mirror): mirror = int(mirror[0]) - mirrorList = getMirrors() + mirrorList = getMirrorList() if mirror not in range(1, len(mirrorList) + 1): print('mirror index is wrong, please check with "sisyphus mirror list"') else: @@ -469,10 +349,7 @@ def setRepo(mirror): mirrorList[i]['isActive'] = True else: mirrorList[i]['isActive'] = False - setActiveMirror(mirrorList) - -# display help menu (CLI frontend) - + writeMirrorCfg(mirrorList) def showHelp(): print("\nUsage : sisyphus command [package(s)] || [file(s)]\n") -- cgit v1.2.3