From a2f57eaa9297cc33e1e3c8fb070c079f2a493db5 Mon Sep 17 00:00:00 2001 From: V3n3RiX Date: Tue, 20 Nov 2018 16:58:32 +0000 Subject: libsisyphus.py : apply the same trick to speed up remote database retrieval --- src/backend/libsisyphus.py | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/backend/libsisyphus.py b/src/backend/libsisyphus.py index fc91131..ebd64c0 100644 --- a/src/backend/libsisyphus.py +++ b/src/backend/libsisyphus.py @@ -47,23 +47,16 @@ def getBinhostURL(): binhostURL = str(portageOutput.rstrip().split("=")[1].strip('\"')) return binhostURL -def getRemotePkgsURL(): - remotePkgsURL = [] +def getCsvUrl(): + remotePkgCsv = [] + remoteDescCsv = [] portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): if "PORTAGE_BINHOST" in portageOutput.rstrip(): - remotePkgsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remotePackagesPre.csv') - return remotePkgsURL - -def getRemoteDscsURL(): - remoteDscsURL = [] - portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE) - - for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"): - if "PORTAGE_BINHOST" in portageOutput.rstrip(): - remoteDscsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv') - return remoteDscsURL + remotePkgCsv = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remotePackagesPre.csv') + remoteDescCsv = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv') + return remotePkgCsv,remoteDescCsv @animation.wait('resolving dependencies') def getPackageDeps(pkgList): @@ -98,14 +91,13 @@ def getWorldDeps(): return binaryDeps,sourceDeps def fetchRemoteDatabase(): - remotePkgsURL = getRemotePkgsURL() - remoteDscsURL = getRemoteDscsURL() + remotePkgCsv,remoteDescCsv = getCsvUrl() http = urllib3.PoolManager() - with http.request('GET', remotePkgsURL, preload_content=False) as tmp_buffer, open(remotePkgsDB, 'wb') as output_file: + with http.request('GET', remotePkgCsv, preload_content=False) as tmp_buffer, open(remotePkgsDB, 'wb') as output_file: shutil.copyfileobj(tmp_buffer, output_file) - with http.request('GET', remoteDscsURL, preload_content=False) as tmp_buffer, open(remoteDscsDB, 'wb') as output_file: + with http.request('GET', remoteDescCsv, preload_content=False) as tmp_buffer, open(remoteDscsDB, 'wb') as output_file: shutil.copyfileobj(tmp_buffer, output_file) def makeLocalDatabase(): @@ -156,15 +148,14 @@ def syncPortageCfg(): def syncAll(): checkRoot() - remotePkgsURL = getRemotePkgsURL() - remoteDscsURL = getRemoteDscsURL() + remotePkgCsv,remoteDescCsv = getCsvUrl() http = urllib3.PoolManager() - reqRemotePkgsTS = http.request('HEAD', remotePkgsURL) + reqRemotePkgsTS = http.request('HEAD', remotePkgCsv) remotePkgsTS = int(parser.parse(reqRemotePkgsTS.headers['last-modified']).strftime("%s")) localPkgsTS = int(os.path.getctime(remotePkgsDB)) - reqRemoteDscsTS = http.request('HEAD', remoteDscsURL) + reqRemoteDscsTS = http.request('HEAD', remoteDescCsv) remoteDscsTS = int(parser.parse(reqRemoteDscsTS.headers['last-modified']).strftime("%s")) localDscsTS = int(os.path.getctime(remoteDscsDB)) -- cgit v1.2.3