From f07b25a5c57a1b6d565c8c4a63b8286ae48d3dc7 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain@orangesquash.org.uk>
Date: Fri, 8 Jul 2016 09:27:25 +0100
Subject: [PATCH 01/13] Support downloading files from a mirror, instead of
 having them local

---
 source/backends/debian/debpkg.d      | 18 ++++++---
 source/backends/debian/debpkgindex.d | 59 ++++++++++++++++++++++++----
 source/utils.d                       | 37 +++++++++++++++++
 3 files changed, 102 insertions(+), 12 deletions(-)

diff --git a/source/backends/debian/debpkg.d b/source/backends/debian/debpkg.d
index dd71dd9a..d1620185 100644
--- a/source/backends/debian/debpkg.d
+++ b/source/backends/debian/debpkg.d
@@ -21,6 +21,7 @@ module ag.backend.debian.debpkg;
 
 import std.stdio;
 import std.string;
+import std.path;
 import std.array : empty, appender;
 import std.file : rmdirRecurse, mkdirRecurse;
 static import std.file;
@@ -28,6 +29,7 @@ import ag.config;
 import ag.archive;
 import ag.backend.intf;
 import ag.logging;
+import ag.utils : isRemote, downloadFile;
 
 
 class DebPackage : Package
@@ -56,7 +58,17 @@ public:
     @property override const(string[string]) description () const { return desc; }
 
     override
-    @property string filename () const { return debFname; }
+    @property string filename () const {
+        if (debFname.isRemote) {
+            immutable auto path = buildPath (tmpDir, debFname.baseName);
+
+            /* XXX: Retry? */
+            downloadFile (debFname, path);
+
+            return path;
+        }
+        return debFname;
+    }
     @property void   filename (string fname) { debFname = fname; }
 
     override
@@ -65,8 +77,6 @@ public:
 
     this (string pname, string pver, string parch)
     {
-        import std.path;
-
         pkgname = pname;
         pkgver = pver;
         pkgarch = parch;
@@ -94,7 +104,6 @@ public:
         auto pa = new ArchiveDecompressor ();
         if (!dataArchive) {
             import std.regex;
-            import std.path;
 
             // extract the payload to a temporary location first
             pa.open (this.filename);
@@ -119,7 +128,6 @@ public:
         auto ca = new ArchiveDecompressor ();
         if (!controlArchive) {
             import std.regex;
-            import std.path;
 
             // extract the payload to a temporary location first
             ca.open (this.filename);
diff --git a/source/backends/debian/debpkgindex.d b/source/backends/debian/debpkgindex.d
index 974b7d29..59895d98 100644
--- a/source/backends/debian/debpkgindex.d
+++ b/source/backends/debian/debpkgindex.d
@@ -30,7 +30,8 @@ import ag.logging;
 import ag.backend.intf;
 import ag.backend.debian.tagfile;
 import ag.backend.debian.debpkg;
-import ag.utils : escapeXml;
+import ag.config;
+import ag.utils : escapeXml, isRemote, downloadFile;
 
 
 class DebianPackageIndex : PackageIndex
@@ -40,14 +41,18 @@ private:
     string rootDir;
     Package[][string] pkgCache;
     bool[string] indexChanged;
+    string tmpDir;
 
 public:
 
     this (string dir)
     {
         this.rootDir = dir;
-        if (!std.file.exists (dir))
+        if (!dir.isRemote && !std.file.exists (dir))
             throw new Exception ("Directory '%s' does not exist.".format (dir));
+
+        auto conf = Config.get ();
+        tmpDir = buildPath (conf.getTmpDir (), dir.baseName);
     }
 
     void release ()
@@ -58,7 +63,19 @@ public:
 
     private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section)
     {
-        auto enDescFname = buildPath (rootDir, "dists", suite, section, "i18n", "Translation-en.bz2");
+        immutable auto enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.xz");
+
+        string enDescFname;
+
+        if (rootDir.isRemote) {
+            enDescFname = buildPath (tmpDir, enDescPath);
+            immutable auto uri = buildPath (rootDir, enDescPath);
+
+            downloadFile (uri, enDescFname);
+        } else {
+            enDescFname = buildPath (rootDir, enDescPath);
+        }
+
         if (!std.file.exists (enDescFname)) {
             logDebug ("No long descriptions for %s/%s", suite, section);
             return;
@@ -119,10 +136,38 @@ public:
 
     private string getIndexFile (string suite, string section, string arch)
     {
-        immutable binDistsPath = buildPath (rootDir, "dists", suite, section, "binary-%s".format (arch));
-        auto indexFname = buildPath (binDistsPath, "Packages.gz");
-        if (!std.file.exists (indexFname))
-            indexFname = buildPath (binDistsPath, "Packages.xz");
+        immutable path = buildPath ("dists", suite, section, "binary-%s".format (arch));
+        immutable binDistsPath = buildPath (rootDir, path);
+
+        string indexFname;
+
+        if (rootDir.isRemote) {
+            import std.file;
+            import std.net.curl : CurlException;
+
+            foreach (string ext; ["xz", "gz"]) {
+                try {
+                    indexFname = buildPath (tmpDir, path, format ("Packages.%s", ext));
+                    immutable auto uri = buildPath (binDistsPath, format ("Packages.%s", ext));
+
+                    /* This should use download(), but that doesn't throw errors */
+                    downloadFile (uri, indexFname);
+
+                    break;
+                } catch (CurlException ex) {
+                    logDebug ("Couldn't download: %s", ex.msg);
+                    indexFname = null;
+                }
+
+                if (indexFname.empty)
+                    throw new Exception (format ("Couldn't download index file for %s/%s/%s", suite, section, arch));
+            }
+        } else {
+            indexFname = buildPath (binDistsPath, "Packages.gz");
+            if (!std.file.exists (indexFname))
+                indexFname = buildPath (binDistsPath, "Packages.xz");
+        }
+
         return indexFname;
     }
 
diff --git a/source/utils.d b/source/utils.d
index dff0d969..d482c715 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -19,6 +19,8 @@
 
 module ag.utils;
 
+import ag.logging;
+
 import std.stdio : writeln;
 import std.string;
 import std.ascii : letters, digits;
@@ -307,6 +309,38 @@ ubyte[] stringArrayToByteArray (string[] strArray) pure @trusted
     return res.data;
 }
 
+@safe
+bool isRemote (const string uri)
+{
+    import std.regex;
+
+    auto uriregex = ctRegex!(`^(https?|ftps?)://`);
+
+    auto match = matchFirst(uri, uriregex);
+
+    return (!match.empty);
+}
+
+void downloadFile (const string url, const string dest)
+{
+    import std.file;
+    import std.path;
+    import std.net.curl;
+
+    if (dest.exists) {
+        logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);
+        return;
+    }
+
+    mkdirRecurse (dest.dirName);
+
+    logDebug ("Downloading %s", url);
+    auto contents = get!(AutoProtocol, ubyte) (url);
+    logDebug ("Downloaded %s", url);
+
+    std.file.write (dest, contents);
+}
+
 unittest
 {
     writeln ("TEST: ", "GCID");
@@ -327,4 +361,7 @@ unittest
     assert (ImageSize (48) < ImageSize (64));
 
     assert (stringArrayToByteArray (["A", "b", "C", "ö", "8"]) == [65, 98, 67, 195, 182, 56]);
+
+    assert (isRemote ("http://test.com"));
+    assert (!isRemote ("/srv/"));
 }

From c8921854c9fc4a95219fe232a9b1c1c98b11c2e9 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain@orangesquash.org.uk>
Date: Sat, 9 Jul 2016 16:23:21 +0100
Subject: [PATCH 02/13] Try to use HTTP/FTP directly so that we can set
 timeouts

---
 source/utils.d | 46 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/source/utils.d b/source/utils.d
index d482c715..f7279af0 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -321,11 +321,35 @@ bool isRemote (const string uri)
     return (!match.empty);
 }
 
+private int onProgressCb (size_t dlTotal,
+                          size_t dlNow,
+                          size_t ulTotal,
+                          size_t ulNow)
+{
+    logDebug ("Got %d/%d", dlNow, dlTotal);
+
+    return 0;
+}
+
+private ulong onReceiveCb (string dest, ubyte[] data)
+{
+    import std.file;
+
+    logDebug ("Downloaded %d bytes", data.length);
+    std.file.write (dest, data);
+
+    return data.length;
+}
+
 void downloadFile (const string url, const string dest)
 {
+    import core.time;
+
     import std.file;
-    import std.path;
     import std.net.curl;
+    import std.path;
+
+    assert (isRemote (url));
 
     if (dest.exists) {
         logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);
@@ -334,11 +358,23 @@ void downloadFile (const string url, const string dest)
 
     mkdirRecurse (dest.dirName);
 
+    /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */
     logDebug ("Downloading %s", url);
-    auto contents = get!(AutoProtocol, ubyte) (url);
-    logDebug ("Downloaded %s", url);
-
-    std.file.write (dest, contents);
+    if (url.startsWith ("http")) {
+        auto downloader = HTTP (url);
+        downloader.onProgress = (dt, dn, ut, un) => onProgressCb (dt, dn, ut, un);
+        downloader.connectTimeout = dur!"seconds" (30);
+        downloader.dataTimeout = dur!"seconds" (30);
+        downloader.onReceive = (data) => onReceiveCb (dest, data);
+        downloader.perform();
+    } else {
+        auto downloader = FTP (url);
+        downloader.onProgress = (dt, dn, ut, un) => onProgressCb (dt, dn, ut, un);
+        downloader.connectTimeout = dur!"seconds" (30);
+        downloader.dataTimeout = dur!"seconds" (30);
+        downloader.onReceive = (data) => onReceiveCb (dest, data);
+        downloader.perform();
+    }
 }
 
 unittest

From 022bd8dd4d24a52220edeeb6fd8f6009493b4e5e Mon Sep 17 00:00:00 2001
From: Iain Lane <iain@orangesquash.org.uk>
Date: Sat, 9 Jul 2016 16:47:39 +0100
Subject: [PATCH 03/13] Try to use std.stdio

---
 source/utils.d | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/source/utils.d b/source/utils.d
index f7279af0..6426e614 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -21,7 +21,7 @@ module ag.utils;
 
 import ag.logging;
 
-import std.stdio : writeln;
+import std.stdio : File, write, writeln;
 import std.string;
 import std.ascii : letters, digits;
 import std.conv : to;
@@ -331,12 +331,10 @@ private int onProgressCb (size_t dlTotal,
     return 0;
 }
 
-private ulong onReceiveCb (string dest, ubyte[] data)
+private ulong onReceiveCb (File f, ubyte[] data)
 {
-    import std.file;
-
     logDebug ("Downloaded %d bytes", data.length);
-    std.file.write (dest, data);
+    f.write (data);
 
     return data.length;
 }
@@ -360,19 +358,20 @@ void downloadFile (const string url, const string dest)
 
     /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */
     logDebug ("Downloading %s", url);
+    auto f = File (dest, "w");
     if (url.startsWith ("http")) {
         auto downloader = HTTP (url);
         downloader.onProgress = (dt, dn, ut, un) => onProgressCb (dt, dn, ut, un);
         downloader.connectTimeout = dur!"seconds" (30);
         downloader.dataTimeout = dur!"seconds" (30);
-        downloader.onReceive = (data) => onReceiveCb (dest, data);
+        downloader.onReceive = (data) => onReceiveCb (f, data);
         downloader.perform();
     } else {
         auto downloader = FTP (url);
         downloader.onProgress = (dt, dn, ut, un) => onProgressCb (dt, dn, ut, un);
         downloader.connectTimeout = dur!"seconds" (30);
         downloader.dataTimeout = dur!"seconds" (30);
-        downloader.onReceive = (data) => onReceiveCb (dest, data);
+        downloader.onReceive = (data) => onReceiveCb (f, data);
         downloader.perform();
     }
 }

From debde9bbef2fb19ad372edb1c0b1dac406a6272a Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Sat, 9 Jul 2016 17:08:18 +0100
Subject: [PATCH 04/13] Be less noisy

---
 source/utils.d | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/source/utils.d b/source/utils.d
index 6426e614..a68d1d0f 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -321,19 +321,8 @@ bool isRemote (const string uri)
     return (!match.empty);
 }
 
-private int onProgressCb (size_t dlTotal,
-                          size_t dlNow,
-                          size_t ulTotal,
-                          size_t ulNow)
-{
-    logDebug ("Got %d/%d", dlNow, dlTotal);
-
-    return 0;
-}
-
 private ulong onReceiveCb (File f, ubyte[] data)
 {
-    logDebug ("Downloaded %d bytes", data.length);
     f.write (data);
 
     return data.length;
@@ -361,19 +350,19 @@ void downloadFile (const string url, const string dest)
     auto f = File (dest, "w");
     if (url.startsWith ("http")) {
         auto downloader = HTTP (url);
-        downloader.onProgress = (dt, dn, ut, un) => onProgressCb (dt, dn, ut, un);
         downloader.connectTimeout = dur!"seconds" (30);
         downloader.dataTimeout = dur!"seconds" (30);
         downloader.onReceive = (data) => onReceiveCb (f, data);
         downloader.perform();
     } else {
         auto downloader = FTP (url);
-        downloader.onProgress = (dt, dn, ut, un) => onProgressCb (dt, dn, ut, un);
         downloader.connectTimeout = dur!"seconds" (30);
         downloader.dataTimeout = dur!"seconds" (30);
         downloader.onReceive = (data) => onReceiveCb (f, data);
         downloader.perform();
     }
+    logDebug ("Downloaded %s", url);
+    f.close();
 }
 
 unittest

From 8ee45b3fd350c173726e09b65ea4066a1dd15d59 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Tue, 26 Jul 2016 16:28:27 +0100
Subject: [PATCH 05/13] Add a helper to find the first {xz, bz2, gz} file we
 can, and download it if necessary

---
 source/backends/debian/debpkgindex.d | 52 ++++------------------------
 source/utils.d                       | 30 ++++++++++++++++
 2 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/source/backends/debian/debpkgindex.d b/source/backends/debian/debpkgindex.d
index 59895d98..f5f6979a 100644
--- a/source/backends/debian/debpkgindex.d
+++ b/source/backends/debian/debpkgindex.d
@@ -31,7 +31,7 @@ import ag.backend.intf;
 import ag.backend.debian.tagfile;
 import ag.backend.debian.debpkg;
 import ag.config;
-import ag.utils : escapeXml, isRemote, downloadFile;
+import ag.utils : escapeXml, isRemote, downloadIfNecessary;
 
 
 class DebianPackageIndex : PackageIndex
@@ -63,20 +63,12 @@ public:
 
     private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section)
     {
-        immutable auto enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.xz");
-
+        immutable auto enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.%s");
         string enDescFname;
 
-        if (rootDir.isRemote) {
-            enDescFname = buildPath (tmpDir, enDescPath);
-            immutable auto uri = buildPath (rootDir, enDescPath);
-
-            downloadFile (uri, enDescFname);
-        } else {
-            enDescFname = buildPath (rootDir, enDescPath);
-        }
-
-        if (!std.file.exists (enDescFname)) {
+        try {
+            enDescFname = downloadIfNecessary (rootDir, tmpDir, enDescPath);
+        } catch (Exception ex) {
             logDebug ("No long descriptions for %s/%s", suite, section);
             return;
         }
@@ -136,39 +128,9 @@ public:
 
     private string getIndexFile (string suite, string section, string arch)
     {
-        immutable path = buildPath ("dists", suite, section, "binary-%s".format (arch));
-        immutable binDistsPath = buildPath (rootDir, path);
-
-        string indexFname;
-
-        if (rootDir.isRemote) {
-            import std.file;
-            import std.net.curl : CurlException;
-
-            foreach (string ext; ["xz", "gz"]) {
-                try {
-                    indexFname = buildPath (tmpDir, path, format ("Packages.%s", ext));
-                    immutable auto uri = buildPath (binDistsPath, format ("Packages.%s", ext));
-
-                    /* This should use download(), but that doesn't throw errors */
-                    downloadFile (uri, indexFname);
-
-                    break;
-                } catch (CurlException ex) {
-                    logDebug ("Couldn't download: %s", ex.msg);
-                    indexFname = null;
-                }
-
-                if (indexFname.empty)
-                    throw new Exception (format ("Couldn't download index file for %s/%s/%s", suite, section, arch));
-            }
-        } else {
-            indexFname = buildPath (binDistsPath, "Packages.gz");
-            if (!std.file.exists (indexFname))
-                indexFname = buildPath (binDistsPath, "Packages.xz");
-        }
+        immutable auto path = buildPath ("dists", suite, section, "binary-%s".format (arch));
 
-        return indexFname;
+        return downloadIfNecessary (rootDir, tmpDir, buildPath (path, "Packages.%s"));
     }
 
     private DebPackage[] loadPackages (string suite, string section, string arch)
diff --git a/source/utils.d b/source/utils.d
index a68d1d0f..9a0cdb92 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -365,6 +365,36 @@ void downloadFile (const string url, const string dest)
     f.close();
 }
 
+string downloadIfNecessary (const string prefix, const string destPrefix, const string suffix)
+{
+    import std.net.curl;
+    import std.path;
+
+    immutable string[] exts = ["xz", "bz2", "gz"];
+    foreach (immutable string ext; exts) {
+        immutable string fileName = format (buildPath (prefix, suffix), ext);
+        immutable string destFileName = format (buildPath (destPrefix, suffix), ext);
+
+        if (fileName.isRemote) {
+            try {
+                /* This should use download(), but that doesn't throw errors */
+                downloadFile (fileName, destFileName);
+
+                return destFileName;
+            } catch (CurlException ex) {
+                logDebug ("Couldn't download: %s", ex.msg);
+            }
+        } else {
+            if (std.file.exists (fileName))
+                return fileName;
+        }
+    }
+
+    /* all extensions failed, so we failed */
+    throw new Exception (format ("Couldn't obtain any file matching %s",
+                         buildPath (prefix, suffix)));
+}
+
 unittest
 {
     writeln ("TEST: ", "GCID");

From 468c394e7d5ebb141b242fc9b30a934c8ef3d60c Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 10:01:15 +0100
Subject: [PATCH 06/13] utils: Use rawWrite when downloading

Otherwise the textual representation of the byte array is written -> [1,
2, 3, 4]
---
 source/utils.d | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/utils.d b/source/utils.d
index 9a0cdb92..837333d0 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -323,7 +323,7 @@ bool isRemote (const string uri)
 
 private ulong onReceiveCb (File f, ubyte[] data)
 {
-    f.write (data);
+    f.rawWrite (data);
 
     return data.length;
 }
@@ -347,7 +347,7 @@ void downloadFile (const string url, const string dest)
 
     /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */
     logDebug ("Downloading %s", url);
-    auto f = File (dest, "w");
+    auto f = File (dest, "wb");
     if (url.startsWith ("http")) {
         auto downloader = HTTP (url);
         downloader.connectTimeout = dur!"seconds" (30);

From 9b3070f5ac78b99532bff8c3ad046b95a6a92fcb Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 10:26:57 +0100
Subject: [PATCH 07/13] immutable things are immutable auto by default

---
 source/backends/debian/debpkg.d      | 2 +-
 source/backends/debian/debpkgindex.d | 4 ++--
 source/utils.d                       | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/backends/debian/debpkg.d b/source/backends/debian/debpkg.d
index d1620185..b4c430da 100644
--- a/source/backends/debian/debpkg.d
+++ b/source/backends/debian/debpkg.d
@@ -60,7 +60,7 @@ public:
     override
     @property string filename () const {
         if (debFname.isRemote) {
-            immutable auto path = buildPath (tmpDir, debFname.baseName);
+            immutable path = buildPath (tmpDir, debFname.baseName);
 
             /* XXX: Retry? */
             downloadFile (debFname, path);
diff --git a/source/backends/debian/debpkgindex.d b/source/backends/debian/debpkgindex.d
index f5f6979a..5bafbdc2 100644
--- a/source/backends/debian/debpkgindex.d
+++ b/source/backends/debian/debpkgindex.d
@@ -63,7 +63,7 @@ public:
 
     private void loadPackageLongDescs (DebPackage[string] pkgs, string suite, string section)
     {
-        immutable auto enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.%s");
+        immutable enDescPath = buildPath ("dists", suite, section, "i18n", "Translation-en.%s");
         string enDescFname;
 
         try {
@@ -128,7 +128,7 @@ public:
 
     private string getIndexFile (string suite, string section, string arch)
     {
-        immutable auto path = buildPath ("dists", suite, section, "binary-%s".format (arch));
+        immutable path = buildPath ("dists", suite, section, "binary-%s".format (arch));
 
         return downloadIfNecessary (rootDir, tmpDir, buildPath (path, "Packages.%s"));
     }
diff --git a/source/utils.d b/source/utils.d
index 837333d0..ec516799 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -370,10 +370,10 @@ string downloadIfNecessary (const string prefix, const string destPrefix, const
     import std.net.curl;
     import std.path;
 
-    immutable string[] exts = ["xz", "bz2", "gz"];
-    foreach (immutable string ext; exts) {
-        immutable string fileName = format (buildPath (prefix, suffix), ext);
-        immutable string destFileName = format (buildPath (destPrefix, suffix), ext);
+    immutable exts = ["xz", "bz2", "gz"];
+    foreach (ref ext; exts) {
+        immutable fileName = format (buildPath (prefix, suffix), ext);
+        immutable destFileName = format (buildPath (destPrefix, suffix), ext);
 
         if (fileName.isRemote) {
             try {

From 1cecf19c26163aabeb9e074e7730d5092306e43d Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 12:25:05 +0100
Subject: [PATCH 08/13] Move downloadIfNecessary to the debian backend

---
 source/backends/debian/debpkgindex.d |  3 +-
 source/backends/debian/utils.d       | 58 ++++++++++++++++++++++++++++
 source/utils.d                       | 30 --------------
 3 files changed, 60 insertions(+), 31 deletions(-)
 create mode 100644 source/backends/debian/utils.d

diff --git a/source/backends/debian/debpkgindex.d b/source/backends/debian/debpkgindex.d
index 5bafbdc2..5dbb57a2 100644
--- a/source/backends/debian/debpkgindex.d
+++ b/source/backends/debian/debpkgindex.d
@@ -30,8 +30,9 @@ import ag.logging;
 import ag.backend.intf;
 import ag.backend.debian.tagfile;
 import ag.backend.debian.debpkg;
+import ag.backend.debian.utils;
 import ag.config;
-import ag.utils : escapeXml, isRemote, downloadIfNecessary;
+import ag.utils : escapeXml, isRemote;
 
 
 class DebianPackageIndex : PackageIndex
diff --git a/source/backends/debian/utils.d b/source/backends/debian/utils.d
new file mode 100644
index 00000000..73d610c1
--- /dev/null
+++ b/source/backends/debian/utils.d
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2016 Canonical Ltd
+ * Author(s): Iain Lane <iain@orangesquash.org.uk>
+ *
+ * Licensed under the GNU Lesser General Public License Version 3
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the license, or
+ * (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this software.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+module ag.backend.debian.utils;
+
+import std.string;
+
+import ag.logging;
+import ag.utils : downloadFile, isRemote;
+
+immutable (string) downloadIfNecessary (const string prefix,
+                                        const string destPrefix,
+                                        const string suffix)
+{
+    import std.net.curl;
+    import std.path;
+
+    immutable exts = ["xz", "bz2", "gz"];
+    foreach (ref ext; exts) {
+        immutable fileName = format (buildPath (prefix, suffix), ext);
+        immutable destFileName = format (buildPath (destPrefix, suffix), ext);
+
+        if (fileName.isRemote) {
+            try {
+                /* This should use download(), but that doesn't throw errors */
+                downloadFile (fileName, destFileName);
+
+                return destFileName;
+            } catch (CurlException ex) {
+                logDebug ("Couldn't download: %s", ex.msg);
+            }
+        } else {
+            if (std.file.exists (fileName))
+                return fileName;
+        }
+    }
+
+    /* all extensions failed, so we failed */
+    throw new Exception (format ("Couldn't obtain any file matching %s",
+                         buildPath (prefix, suffix)));
+}
diff --git a/source/utils.d b/source/utils.d
index ec516799..bb4cce4d 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -365,36 +365,6 @@ void downloadFile (const string url, const string dest)
     f.close();
 }
 
-string downloadIfNecessary (const string prefix, const string destPrefix, const string suffix)
-{
-    import std.net.curl;
-    import std.path;
-
-    immutable exts = ["xz", "bz2", "gz"];
-    foreach (ref ext; exts) {
-        immutable fileName = format (buildPath (prefix, suffix), ext);
-        immutable destFileName = format (buildPath (destPrefix, suffix), ext);
-
-        if (fileName.isRemote) {
-            try {
-                /* This should use download(), but that doesn't throw errors */
-                downloadFile (fileName, destFileName);
-
-                return destFileName;
-            } catch (CurlException ex) {
-                logDebug ("Couldn't download: %s", ex.msg);
-            }
-        } else {
-            if (std.file.exists (fileName))
-                return fileName;
-        }
-    }
-
-    /* all extensions failed, so we failed */
-    throw new Exception (format ("Couldn't obtain any file matching %s",
-                         buildPath (prefix, suffix)));
-}
-
 unittest
 {
     writeln ("TEST: ", "GCID");

From 6d97045f1b60c5b7133c3cf2ae143bae4012a5c5 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 14:57:34 +0100
Subject: [PATCH 09/13] Add a documentation comment

---
 source/backends/debian/utils.d | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/source/backends/debian/utils.d b/source/backends/debian/utils.d
index 73d610c1..a53c0f5f 100644
--- a/source/backends/debian/utils.d
+++ b/source/backends/debian/utils.d
@@ -25,6 +25,21 @@ import std.string;
 import ag.logging;
 import ag.utils : downloadFile, isRemote;
 
+/**
+ * If prefix is remote, download prefix + suffix, otherwise check if prefix +
+ * suffix exists.
+ *
+ * Returns: Path to the file, which is guaranteed to exist.
+ *
+ * Params:
+ *      prefix = First part of the address, i.e.
+ *               "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/"
+ *      destPrefix = If the file is remote, the directory to save it under,
+ *                   which is created if necessary.
+ *      suffix = the rest of the address, so that prefix + suffix is a full
+ *               path or URL, i.e.
+ *               "dists/unstable/main/binary-i386/Packages.xz"
+ */
 immutable (string) downloadIfNecessary (const string prefix,
                                         const string destPrefix,
                                         const string suffix)

From 55454049e8eac0d82e6cd08397df018ccca349f8 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 15:27:36 +0100
Subject: [PATCH 10/13] Make downloadFile support retrying on timeouts

Retry 5 times by default.
---
 source/utils.d | 54 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 16 deletions(-)

diff --git a/source/utils.d b/source/utils.d
index bb4cce4d..4f92d2ca 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -328,7 +328,15 @@ private ulong onReceiveCb (File f, ubyte[] data)
     return data.length;
 }
 
-void downloadFile (const string url, const string dest)
+/**
+ * Download `url` to `dest`.
+ *
+ * Params:
+ *      url = The URL to download.
+ *      dest = The location for the downloaded file.
+ *      retryCount = Number of times to retry on timeout.
+ */
+void downloadFile (const string url, const string dest, const uint retryCount = 5)
 {
     import core.time;
 
@@ -347,22 +355,36 @@ void downloadFile (const string url, const string dest)
 
     /* the curl library is stupid; you can't make an AutoProtocol to set timeouts */
     logDebug ("Downloading %s", url);
-    auto f = File (dest, "wb");
-    if (url.startsWith ("http")) {
-        auto downloader = HTTP (url);
-        downloader.connectTimeout = dur!"seconds" (30);
-        downloader.dataTimeout = dur!"seconds" (30);
-        downloader.onReceive = (data) => onReceiveCb (f, data);
-        downloader.perform();
-    } else {
-        auto downloader = FTP (url);
-        downloader.connectTimeout = dur!"seconds" (30);
-        downloader.dataTimeout = dur!"seconds" (30);
-        downloader.onReceive = (data) => onReceiveCb (f, data);
-        downloader.perform();
+    try {
+        auto f = File (dest, "wb");
+        scope(exit) f.close();
+        scope(failure) remove(dest);
+
+        if (url.startsWith ("http")) {
+            auto downloader = HTTP (url);
+            downloader.connectTimeout = dur!"seconds" (30);
+            downloader.dataTimeout = dur!"seconds" (30);
+            downloader.onReceive = (data) => onReceiveCb (f, data);
+            downloader.perform();
+        } else {
+            auto downloader = FTP (url);
+            downloader.connectTimeout = dur!"seconds" (30);
+            downloader.dataTimeout = dur!"seconds" (30);
+            downloader.onReceive = (data) => onReceiveCb (f, data);
+            downloader.perform();
+        }
+        logDebug ("Downloaded %s", url);
+    } catch (CurlTimeoutException e) {
+        if (retryCount > 0) {
+            logDebug ("Failed to download %s, will retry %d more %s",
+                      url,
+                      retryCount,
+                      retryCount > 1 ? "times" : "time");
+            downloadFile (url, dest, retryCount - 1);
+        } else {
+            throw e;
+        }
     }
-    logDebug ("Downloaded %s", url);
-    f.close();
 }
 
 unittest

From 26b5fd0b76bcfa0b189b3de1070c5488a44df95e Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 16:26:48 +0100
Subject: [PATCH 11/13] utils: Add pre and postconditions to downloadFile

---
 source/utils.d | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/source/utils.d b/source/utils.d
index 4f92d2ca..d51519bc 100644
--- a/source/utils.d
+++ b/source/utils.d
@@ -337,6 +337,15 @@ private ulong onReceiveCb (File f, ubyte[] data)
  *      retryCount = Number of times to retry on timeout.
  */
 void downloadFile (const string url, const string dest, const uint retryCount = 5)
+in
+{
+    assert (isRemote (url));
+}
+out
+{
+    assert (std.file.exists (dest));
+}
+body
 {
     import core.time;
 
@@ -344,7 +353,6 @@ void downloadFile (const string url, const string dest, const uint retryCount =
     import std.net.curl;
     import std.path;
 
-    assert (isRemote (url));
 
     if (dest.exists) {
         logDebug ("Already downloaded '%s' into '%s', won't redownload", url, dest);

From dd61afacbbd1166f1a2ed2fa5d3e4b8a6059a514 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 16:51:35 +0100
Subject: [PATCH 12/13] Remove a comment asking for something which now happens

---
 source/backends/debian/debpkg.d | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/backends/debian/debpkg.d b/source/backends/debian/debpkg.d
index b4c430da..9fd23f96 100644
--- a/source/backends/debian/debpkg.d
+++ b/source/backends/debian/debpkg.d
@@ -62,7 +62,6 @@ public:
         if (debFname.isRemote) {
             immutable path = buildPath (tmpDir, debFname.baseName);
 
-            /* XXX: Retry? */
             downloadFile (debFname, path);
 
             return path;

From 5494463a8467beb1fe93eaac69e3754c797b5237 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Wed, 27 Jul 2016 17:53:37 +0100
Subject: [PATCH 13/13] debian/utils.d: Update docs for downloadIfNecessary

This function is kind of weird, we might want to think of a better API
for it.
---
 source/backends/debian/utils.d | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/source/backends/debian/utils.d b/source/backends/debian/utils.d
index a53c0f5f..6171bb30 100644
--- a/source/backends/debian/utils.d
+++ b/source/backends/debian/utils.d
@@ -26,8 +26,8 @@ import ag.logging;
 import ag.utils : downloadFile, isRemote;
 
 /**
- * If prefix is remote, download prefix + suffix, otherwise check if prefix +
- * suffix exists.
+ * If prefix is remote, download the first of (prefix + suffix).{xz,bz2,gz},
+ * otherwise check if any of (prefix + suffix).{xz,bz2,gz} exists.
  *
  * Returns: Path to the file, which is guaranteed to exist.
  *
@@ -36,9 +36,11 @@ import ag.utils : downloadFile, isRemote;
  *               "http://ftp.debian.org/debian/" or "/srv/mirrors/debian/"
  *      destPrefix = If the file is remote, the directory to save it under,
  *                   which is created if necessary.
- *      suffix = the rest of the address, so that prefix + suffix is a full
- *               path or URL, i.e.
- *               "dists/unstable/main/binary-i386/Packages.xz"
+ *      suffix = the rest of the address, so that (prefix +
+ *               suffix).format({xz,bz2,gz}) is a full path or URL, i.e.
+ *               "dists/unstable/main/binary-i386/Packages.%s". The suffix must
+ *               contain exactly one "%s"; this function is only suitable for
+ *               finding `.xz`, `.bz2` and `.gz` files.
  */
 immutable (string) downloadIfNecessary (const string prefix,
                                         const string destPrefix,