From 69b4a4f9ac349cc2231b2122ca9d8936bd2c871b Mon Sep 17 00:00:00 2001 From: Rasmus Thomsen Date: Fri, 15 May 2020 14:58:38 +0200 Subject: [PATCH] alpine: add capabilities to download packages via HTTP --- .../backends/alpinelinux/apkindexutils.d | 150 ++++++++++++++++++ src/asgen/backends/alpinelinux/apkpkg.d | 49 +++++- src/asgen/backends/alpinelinux/apkpkgindex.d | 100 ++++-------- src/asgen/meson.build | 1 + 4 files changed, 220 insertions(+), 80 deletions(-) create mode 100644 src/asgen/backends/alpinelinux/apkindexutils.d diff --git a/src/asgen/backends/alpinelinux/apkindexutils.d b/src/asgen/backends/alpinelinux/apkindexutils.d new file mode 100644 index 00000000..cb16636b --- /dev/null +++ b/src/asgen/backends/alpinelinux/apkindexutils.d @@ -0,0 +1,150 @@ +module asgen.backends.alpinelinux.apkindexutils; + +import std.algorithm : remove; +import std.algorithm.iteration : map; +import std.algorithm.searching : canFind; +import std.array : join, split; +import std.file : exists; +import std.format : format; +import std.path : buildPath; +import std.range : empty, InputRange, isForwardRange; +import std.string : splitLines, startsWith, strip; +import std.utf : validate; + +import std.stdio; + +import asgen.backends.alpinelinux.apkpkg; +import asgen.downloader : Downloader, DownloadException; +import asgen.logging : logDebug; +import asgen.utils : isRemote; + +/** +* Struct representing a block inside of an APKINDEX. Each block, seperated by +* a newline, contains information about exactly one package. +*/ +struct ApkIndexBlock { + string arch; + string maintainer; + string pkgname; + string pkgversion; + string pkgdesc; + + @property string archiveName () { + return format ("%s-%s.apk", this.pkgname, this.pkgversion); + } +} + +/** +* Range for looping over the contents of an APKINDEX, block by block. +*/ +struct ApkIndexBlockRange { + this (string contents) + { + this.lines = contents.splitLines; + this.getNextBlock(); + } + + @property ApkIndexBlock front () const { + return this.currentBlock; + } + + @property bool empty () { + return this.m_empty; + } + + void popFront () + { + this.getNextBlock (); + } + + @property ApkIndexBlockRange save () { return this; } + +private: + void getNextBlock () { + string[] completePair; + uint iterations = 0; + + currentBlock = ApkIndexBlock(); + foreach (currentLine; this.lines[this.lineDelta .. $]) { + iterations++; + if (currentLine == "") { + // next block for next package started + break; + } if (currentLine.canFind (":")) { + if (completePair.empty) { + completePair = [currentLine]; + continue; + } + + auto pair = completePair.join (" ").split (":"); + this.setCurrentBlock (pair[0], pair[1]); + completePair = [currentLine]; + } else { + completePair ~= currentLine.strip (); + } + } + + this.lineDelta += iterations; + this.m_empty = this.lineDelta == this.lines.length; + } + + void setCurrentBlock (string key, string value) { + switch (key) { + case "A": + this.currentBlock.arch = value; + break; + case "m": + this.currentBlock.maintainer = value; + break; + case "P": + this.currentBlock.pkgname = value; + break; + case "T": + this.currentBlock.pkgdesc = value; + break; + case "V": + this.currentBlock.pkgversion = value; + break; + default: + // We dont care about other keys + break; + } + } + + string[] lines; + ApkIndexBlock currentBlock; + bool m_empty; + uint lineDelta; +} + +static assert (isForwardRange!ApkIndexBlockRange); + +/** + * Download apkindex if required. Returns the path to the local copy of the APKINDEX. + */ +immutable (string) downloadIfNecessary (const string prefix, + const string destPrefix, + const string srcFileName, + const string destFileName) +{ + auto downloader = Downloader.get; + + immutable filePath = buildPath (prefix, srcFileName); + immutable destFilePath = buildPath (destPrefix, destFileName); + + if (filePath.isRemote) { + try { + downloader.downloadFile (filePath, destFilePath); + + return destFilePath; + } catch (DownloadException e) { + logDebug ("Unable to download: %s", e.msg); + } + } else { + if (filePath.exists) + return filePath; + } + + /* all extensions failed, so we failed */ + throw new Exception ("Could not obtain any file matching %s".format (buildPath (prefix, srcFileName))); +} diff --git a/src/asgen/backends/alpinelinux/apkpkg.d b/src/asgen/backends/alpinelinux/apkpkg.d index eeb3a1a6..b4fafaf5 100644 --- a/src/asgen/backends/alpinelinux/apkpkg.d +++ b/src/asgen/backends/alpinelinux/apkpkg.d @@ -22,13 +22,15 @@ module asgen.backends.alpinelinux.apkpkg; -import std.stdio; -import std.string; import std.array : empty; +import std.format : format; +import std.path : baseName, buildNormalizedPath, buildPath; -import asgen.logging; -import asgen.zarchive; import asgen.backends.interfaces; +import asgen.config : Config; +import asgen.downloader : Downloader; +import asgen.utils : isRemote; +import asgen.zarchive : ArchiveDecompressor; final class AlpinePackage : Package { @@ -39,12 +41,24 @@ private: string pkgmaintainer; string[string] desc; string pkgFname; + string localPkgFName; + string tmpDir; - string[] contentsL; + string[] contentsL = null; ArchiveDecompressor archive; public: + this (string pkgname, string pkgver, string pkgarch) + { + this.pkgname = pkgname; + this.pkgver = pkgver; + this.pkgarch = pkgarch; + + auto conf = Config.get (); + this.tmpDir = buildPath (conf.getTmpDir (), format ("%s-%s_%s", name, ver, arch)); + } + override @property string name () const { return this.pkgname; @@ -85,9 +99,23 @@ public: this.pkgFname = fname; } - override @property string getFilename () const + override @property string getFilename () { - return pkgFname; + if (!this.localPkgFName.empty) + return this.localPkgFName; + + if (pkgFname.isRemote) { + synchronized (this) { + auto dl = Downloader.get; + immutable path = buildNormalizedPath (this.tmpDir, this.pkgFname.baseName); + dl.downloadFile (this.pkgFname, path); + this.localPkgFName = path; + return this.localPkgFName; + } + } else { + this.localPkgFName = pkgFname; + return this.localPkgFName; + } } override @property string maintainer () const @@ -115,6 +143,13 @@ public: @property override string[] contents () { + if (!this.contentsL.empty) + return this.contentsL; + + ArchiveDecompressor ad; + ad.open (this.getFilename); + this.contentsL = ad.readContents (); + return this.contentsL; } diff --git a/src/asgen/backends/alpinelinux/apkpkgindex.d b/src/asgen/backends/alpinelinux/apkpkgindex.d index 48570752..a068781d 100644 --- a/src/asgen/backends/alpinelinux/apkpkgindex.d +++ b/src/asgen/backends/alpinelinux/apkpkgindex.d @@ -19,36 +19,41 @@ module asgen.backends.alpinelinux.apkpkgindex; -import std.algorithm : canFind, filter, endsWith, remove; -import std.array : appender, join, split; +import std.array : appender; import std.conv : to; import std.exception : enforce; -import std.file : dirEntries, exists, SpanMode; +import std.file : exists; import std.format : format; import std.path : baseName, buildPath; -import std.range : empty; -import std.string : splitLines, startsWith, strip; -import std.utf : UTFException, validate; +import std.utf : validate; -import asgen.logging; -import asgen.zarchive; -import asgen.utils : escapeXml; +import asgen.config : Config; +import asgen.logging : logError; +import asgen.zarchive : ArchiveDecompressor; +import asgen.utils : escapeXml, isRemote; import asgen.backends.interfaces; import asgen.backends.alpinelinux.apkpkg; +import asgen.backends.alpinelinux.apkindexutils; final class AlpinePackageIndex : PackageIndex { private: string rootDir; + string tmpDir; Package[][string] pkgCache; public: this (string dir) { - enforce (exists (dir), format ("Directory '%s' does not exist.", dir)); + if (!dir.isRemote) + enforce (exists (dir), format ("Directory '%s' does not exist.", dir)); + this.rootDir = dir; + + auto conf = Config.get (); + tmpDir = buildPath (conf.getTmpDir, dir.baseName); } override void release () @@ -65,81 +70,30 @@ public: pkg.setDescription (desc, "C"); } - private void setPkgValues (ref AlpinePackage pkg, string[] keyValueString) - { - immutable key = keyValueString[0].strip; - immutable value = keyValueString[1].strip; - - switch (key) { - case "pkgname": - pkg.name = value; - break; - case "pkgver": - pkg.ver = value; - break; - case "arch": - pkg.arch = value; - break; - case "maintainer": - pkg.maintainer = value; - break; - case "pkgdesc": - setPkgDescription(pkg, value); - break; - default: - // We don't care about other entries - break; - } - } - private Package[] loadPackages (string suite, string section, string arch) { auto apkRootPath = buildPath (rootDir, suite, section, arch); - ArchiveDecompressor ad; + auto indexFPath = downloadIfNecessary(apkRootPath, tmpDir, "APKINDEX.tar.gz", format("APKINDEX-%s-%s-%s.tar.gz", suite, section, arch)); AlpinePackage[string] pkgsMap; + ArchiveDecompressor ad; + ad.open (indexFPath); + auto indexString = cast(string) ad.readData ("APKINDEX"); + validate (indexString); + auto range = ApkIndexBlockRange (indexString); - foreach (packageArchivePath; dirEntries (apkRootPath, SpanMode.shallow).filter!( - f => f.name.endsWith (".apk"))) { - auto fileName = packageArchivePath.baseName (); + foreach (pkgInfo; range) { + auto fileName = pkgInfo.archiveName; AlpinePackage pkg; if (fileName in pkgsMap) { pkg = pkgsMap[fileName]; } else { - pkg = new AlpinePackage (); + pkg = new AlpinePackage (pkgInfo.pkgname, pkgInfo.pkgversion, pkgInfo.arch); pkgsMap[fileName] = pkg; } - ad.open (packageArchivePath); - auto pkgInfoData = cast(string) ad.readData (".PKGINFO"); - - try { - validate (pkgInfoData); - } catch (UTFException e) { - logError ("PKGINFO file in archive %s contained invalid UTF-8, skipping!", - packageArchivePath); - continue; - } - - pkg.filename = packageArchivePath; - auto lines = pkgInfoData.splitLines (); - // If the current line doesn't contain a = it's meant to extend the previous line - string[] completePair; - foreach (currentLine; lines) { - if (currentLine.canFind ("=")) { - if (completePair.empty) { - completePair = [currentLine]; - continue; - } - - this.setPkgValues (pkg, completePair.join (" ").split ("=")); - completePair = [currentLine]; - } else if (!currentLine.startsWith ("#")) { - completePair ~= currentLine.strip.split ("#")[0]; - } - } - // We didn't process the last line yet - this.setPkgValues (pkg, completePair.join (" ").split ("=")); - pkg.contents = ad.readContents ().remove!("a == \".PKGINFO\" || a.startsWith (\".SIGN\")"); + pkg.filename = buildPath(rootDir, suite, section, arch, fileName); + pkg.maintainer = pkgInfo.maintainer; + setPkgDescription(pkg, pkgInfo.pkgdesc); } // perform a sanity check, so we will never emit invalid packages diff --git a/src/asgen/meson.build b/src/asgen/meson.build index afd3ab1d..e137d5af 100644 --- a/src/asgen/meson.build +++ b/src/asgen/meson.build @@ -64,6 +64,7 @@ backend_sources = [ 'backends/dummy/pkgindex.d', 'backends/alpinelinux/package.d', + 'backends/alpinelinux/apkindexutils.d', 'backends/alpinelinux/apkpkg.d', 'backends/alpinelinux/apkpkgindex.d',