diff --git a/source/backends/debian/debpkg.d b/source/backends/debian/debpkg.d index dd71dd9a..9fd23f96 100644 --- a/source/backends/debian/debpkg.d +++ b/source/backends/debian/debpkg.d @@ -21,6 +21,7 @@ module ag.backend.debian.debpkg; import std.stdio; import std.string; +import std.path; import std.array : empty, appender; import std.file : rmdirRecurse, mkdirRecurse; static import std.file; @@ -28,6 +29,7 @@ import ag.config; import ag.archive; import ag.backend.intf; import ag.logging; +import ag.utils : isRemote, downloadFile; class DebPackage : Package @@ -56,7 +58,16 @@ public: @property override const(string[string]) description () const { return desc; } override - @property string filename () const { return debFname; } + @property string filename () const { + if (debFname.isRemote) { + immutable path = buildPath (tmpDir, debFname.baseName); + + downloadFile (debFname, path); + + return path; + } + return debFname; + } @property void filename (string fname) { debFname = fname; } override @@ -65,8 +76,6 @@ public: this (string pname, string pver, string parch) { - import std.path; - pkgname = pname; pkgver = pver; pkgarch = parch; @@ -94,7 +103,6 @@ public: auto pa = new ArchiveDecompressor (); if (!dataArchive) { import std.regex; - import std.path; // extract the payload to a temporary location first pa.open (this.filename); @@ -119,7 +127,6 @@ public: auto ca = new ArchiveDecompressor (); if (!controlArchive) { import std.regex; - import std.path; // extract the payload to a temporary location first ca.open (this.filename); diff --git a/source/backends/debian/debpkgindex.d b/source/backends/debian/debpkgindex.d index 974b7d29..5dbb57a2 100644 --- a/source/backends/debian/debpkgindex.d +++ b/source/backends/debian/debpkgindex.d @@ -30,7 +30,9 @@ import ag.logging; import ag.backend.intf; import ag.backend.debian.tagfile; import ag.backend.debian.debpkg; -import ag.utils : escapeXml; +import ag.backend.debian.utils; +import ag.config; +import ag.utils : escapeXml, isRemote; class DebianPackageIndex : PackageIndex @@ -40,14 +42,18 @@ private: string rootDir; Package[][string] pkgCache; bool[string] indexChanged; + string tmpDir; public: this (string dir) { this.rootDir = dir; - if (!std.file.exists (dir)) + if (!dir.isRemote && !std.file.exists (dir)) throw new Exception ("Directory '%s' does not exist.".format (dir)); + + auto conf = Config.get (); + tmpDir = buildPath (conf.getTmpDir (), dir.baseName); } void release () @@ -58,8 +64,12 @@ public: private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section) { - auto enDescFname = buildPath (rootDir, "dists", suite, section, "i18n", "Translation-en.bz2"); - if (!std.file.exists (enDescFname)) { + immutable enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.%s"); + string enDescFname; + + try { + enDescFname = downloadIfNecessary (rootDir, tmpDir, enDescPath); + } catch (Exception ex) { logDebug ("No long descriptions for %s/%s", suite, section); return; } @@ -119,11 +129,9 @@ public: private string getIndexFile (string suite, string section, string arch) { - immutable binDistsPath = buildPath (rootDir, "dists", suite, section, "binary-%s".format (arch)); - auto indexFname = buildPath (binDistsPath, "Packages.gz"); - if (!std.file.exists (indexFname)) - indexFname = buildPath (binDistsPath, "Packages.xz"); - return indexFname; + immutable path = buildPath ("dists", suite, section, "binary-%s".format (arch)); + + return downloadIfNecessary (rootDir, tmpDir, buildPath (path, "Packages.%s")); } private DebPackage[] loadPackages (string suite, string section, string arch) diff --git a/source/backends/debian/utils.d b/source/backends/debian/utils.d new file mode 100644 index 00000000..6171bb30 --- /dev/null +++ b/source/backends/debian/utils.d @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2016 Canonical Ltd + * Author(s): Iain Lane + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module ag.backend.debian.utils; + +import std.string; + +import ag.logging; +import ag.utils : downloadFile, isRemote; + +/** + * If prefix is remote, download the first of (prefix + suffix).{xz,bz2,gz}, + * otherwise check if any of (prefix + suffix).{xz,bz2,gz} exists. + * + * Returns: Path to the file, which is guaranteed to exist. + * + * Params: + * prefix = First part of the address, i.e. + * "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/" + * destPrefix = If the file is remote, the directory to save it under, + * which is created if necessary. + * suffix = the rest of the address, so that (prefix + + * suffix).format({xz,bz2,gz}) is a full path or URL, i.e. + * "dists/unstable/main/binary-i386/Packages.%s". The suffix must + * contain exactly one "%s"; this function is only suitable for + * finding `.xz`, `.bz2` and `.gz` files. + */ +immutable (string) downloadIfNecessary (const string prefix, + const string destPrefix, + const string suffix) +{ + import std.net.curl; + import std.path; + + immutable exts = ["xz", "bz2", "gz"]; + foreach (ref ext; exts) { + immutable fileName = format (buildPath (prefix, suffix), ext); + immutable destFileName = format (buildPath (destPrefix, suffix), ext); + + if (fileName.isRemote) { + try { + /* This should use download(), but that doesn't throw errors */ + downloadFile (fileName, destFileName); + + return destFileName; + } catch (CurlException ex) { + logDebug ("Couldn't download: %s", ex.msg); + } + } else { + if (std.file.exists (fileName)) + return fileName; + } + } + + /* all extensions failed, so we failed */ + throw new Exception (format ("Couldn't obtain any file matching %s", + buildPath (prefix, suffix))); +} diff --git a/source/utils.d b/source/utils.d index dff0d969..d51519bc 100644 --- a/source/utils.d +++ b/source/utils.d @@ -19,7 +19,9 @@ module ag.utils; -import std.stdio : writeln; +import ag.logging; + +import std.stdio : File, write, writeln; import std.string; import std.ascii : letters, digits; import std.conv : to; @@ -307,6 +309,92 @@ ubyte[] stringArrayToByteArray (string[] strArray) pure @trusted return res.data; } +@safe +bool isRemote (const string uri) +{ + import std.regex; + + auto uriregex = ctRegex!(`^(https?|ftps?)://`); + + auto match = matchFirst(uri, uriregex); + + return (!match.empty); +} + +private ulong onReceiveCb (File f, ubyte[] data) +{ + f.rawWrite (data); + + return data.length; +} + +/** + * Download `url` to `dest`. + * + * Params: + * url = The URL to download. + * dest = The location for the downloaded file. + * retryCount = Number of times to retry on timeout. + */ +void downloadFile (const string url, const string dest, const uint retryCount = 5) +in +{ + assert (isRemote (url)); +} +out +{ + assert (std.file.exists (dest)); +} +body +{ + import core.time; + + import std.file; + import std.net.curl; + import std.path; + + + if (dest.exists) { + logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest); + return; + } + + mkdirRecurse (dest.dirName); + + /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */ + logDebug ("Downloading %s", url); + try { + auto f = File (dest, "wb"); + scope(exit) f.close(); + scope(failure) remove(dest); + + if (url.startsWith ("http")) { + auto downloader = HTTP (url); + downloader.connectTimeout = dur!"seconds" (30); + downloader.dataTimeout = dur!"seconds" (30); + downloader.onReceive = (data) => onReceiveCb (f, data); + downloader.perform(); + } else { + auto downloader = FTP (url); + downloader.connectTimeout = dur!"seconds" (30); + downloader.dataTimeout = dur!"seconds" (30); + downloader.onReceive = (data) => onReceiveCb (f, data); + downloader.perform(); + } + logDebug ("Downloaded %s", url); + } catch (CurlTimeoutException e) { + if (retryCount > 0) { + logDebug ("Failed to download %s, will retry %d more %s", + url, + retryCount, + retryCount > 1 ? "times" : "time"); + downloadFile (url, dest, retryCount - 1); + } else { + throw e; + } + } +} + unittest { writeln ("TEST: ", "GCID"); @@ -327,4 +415,7 @@ unittest assert (ImageSize (48) < ImageSize (64)); assert (stringArrayToByteArray (["A", "b", "C", "รถ", "8"]) == [65, 98, 67, 195, 182, 56]); + + assert (isRemote ("http://test.com")); + assert (!isRemote ("/srv/")); }