From 7861941af08cd1e273f08e834a5690b5e8b78d43 Mon Sep 17 00:00:00 2001 From: Neil Mayhew Date: Fri, 13 May 2016 16:30:24 -0600 Subject: [PATCH 1/3] Ensure gzipped files have repeatable results The gz format puts a timestamp in the compressed file. Therefore repeated runs produce different gz files even though the input data is the same. Use a timestamp of 0 to ensure the compressed data is the same when the input data is the same. Neither the gzip executable nor the libarchive library provide a way to specify the timestamp explicitly, so we set the modification time of the input file. This is going to be removed anyway, so it doesn't matter what it's set to. --- source/archive.d | 14 ++++++++------ source/engine.d | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/archive.d b/source/archive.d index 579b2e51..b299f0bf 100644 --- a/source/archive.d +++ b/source/archive.d @@ -374,10 +374,13 @@ void compressAndSave (ubyte[] data, string fname, ArchiveType atype) void saveCompressed (string fname, ArchiveType atype) { import std.process; + import std.datetime : SysTime, unixTimeToStdTime; Pid pid; File cf; if (atype == ArchiveType.GZIP) { + SysTime epoch = unixTimeToStdTime (0); + std.file.setTimes (fname, epoch, epoch); // Ensure repeatable result cf = File (fname ~ ".gz", "w"); pid = spawnProcess (["gzip", "-c", fname], std.stdio.stdin, cf); } else { @@ -395,6 +398,7 @@ class ArchiveCompressor private: string archiveFname; + ArchiveType archiveType; archive *ar; bool closed; @@ -402,13 +406,8 @@ public: this (ArchiveType type) { + archiveType = type; ar = archive_write_new (); - - if (type == ArchiveType.GZIP) - archive_write_add_filter_gzip (ar); - else - archive_write_add_filter_xz (ar); - archive_write_set_format_pax_restricted (ar); closed = true; } @@ -434,6 +433,9 @@ public: return; archive_write_close (ar); closed = true; + + saveCompressed (archiveFname, archiveType); + std.file.remove (archiveFname); } void addFile (string fname, string dest = null) diff --git a/source/engine.d b/source/engine.d index ed52d3c6..b86d3dad 100644 --- a/source/engine.d +++ b/source/engine.d @@ -228,7 +228,7 @@ public: if (withIconTar) { foreach (size; iconTarSizes) { iconTar[size] = new ArchiveCompressor (ArchiveType.GZIP); - iconTar[size].open (buildPath (dataExportDir, format ("icons-%sx%s.tar.gz", size, size))); + iconTar[size].open (buildPath (dataExportDir, format ("icons-%sx%s.tar", size, size))); } } From d147262541e43709a12b1a2be617f439ca01e295 Mon Sep 17 00:00:00 2001 From: Neil Mayhew Date: Fri, 13 May 2016 17:23:29 -0600 Subject: [PATCH 2/3] Ensure repeatable html results by omitting timestamps --- data/templates/default/issues_index.html | 4 ---- data/templates/default/issues_page.html | 4 ---- data/templates/default/metainfo_index.html | 4 ---- data/templates/default/metainfo_page.html | 4 ---- data/templates/default/section_overview.html | 4 ---- data/templates/default/sections_index.html | 4 ---- 6 files changed, 24 deletions(-) diff --git a/data/templates/default/issues_index.html b/data/templates/default/issues_index.html index 8917e571..b307f027 100644 --- a/data/templates/default/issues_index.html +++ b/data/templates/default/issues_index.html @@ -32,8 +32,4 @@

{{maintainer}}

{{/summaries}} {{/partial}} -{{#partial}}{ float_right } -Last updated on: {{time}} -{{/partial}} - {{> base}} diff --git a/data/templates/default/issues_page.html b/data/templates/default/issues_page.html index 7ad32e8a..fdc09ebf 100644 --- a/data/templates/default/issues_page.html +++ b/data/templates/default/issues_page.html @@ -1,9 +1,5 @@ {{#partial}}{title}Issues for {{package_name}} in {{suite}}/{{section}}{{/partial}} -{{#partial}}{ float_right } -Last updated on: {{time}} -{{/partial}} - {{#partial}}{ header_content } ⇦ | diff --git a/data/templates/default/metainfo_index.html b/data/templates/default/metainfo_index.html index c6ee0c6b..80672f4f 100644 --- a/data/templates/default/metainfo_index.html +++ b/data/templates/default/metainfo_index.html @@ -30,8 +30,4 @@

{{maintainer}}

{{/partial}} -{{#partial}}{ float_right } -Last updated on: {{time}} -{{/partial}} - {{> base}} diff --git a/data/templates/default/metainfo_page.html b/data/templates/default/metainfo_page.html index 73792e9e..784739d7 100644 --- a/data/templates/default/metainfo_page.html +++ b/data/templates/default/metainfo_page.html @@ -1,9 +1,5 @@ {{#partial}}{title}{{package_name}} in {{suite}}/{{section}}{{/partial}} -{{#partial}}{ float_right } -Last updated on: {{time}} -{{/partial}} - {{#partial}}{ head_extra } diff --git a/data/templates/default/section_overview.html b/data/templates/default/section_overview.html index 2b2ba7c5..ba3bafe0 100644 --- a/data/templates/default/section_overview.html +++ b/data/templates/default/section_overview.html @@ -164,8 +164,4 @@

Global data validation result

{{/partial}} -{{#partial}}{ float_right } -Last updated on: {{time}} -{{/partial}} - {{> base}} diff --git a/data/templates/default/sections_index.html b/data/templates/default/sections_index.html index 753e234e..cff8937f 100644 --- a/data/templates/default/sections_index.html +++ b/data/templates/default/sections_index.html @@ -123,8 +123,4 @@

Health of suite "{{suite}}"

{{/partial}} -{{#partial}}{ float_right } -Last updated on: {{time}} -{{/partial}} - {{> base}} From f529b964ca2e424dc1e792470be1a374df1c4b24 Mon Sep 17 00:00:00 2001 From: Neil Mayhew Date: Fri, 13 May 2016 17:27:07 -0600 Subject: [PATCH 3/3] Ensure repeatable hints and icons results by not using parallelism Parallelism gives rise to a non-deterministic ordering of the icons and hints. Removing parallelism makes no appreciable difference to the speed. --- source/engine.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/engine.d b/source/engine.d index b86d3dad..446e2c9d 100644 --- a/source/engine.d +++ b/source/engine.d @@ -233,7 +233,7 @@ public: } // collect metadata, icons and hints for the given packages - foreach (ref pkg; parallel (pkgs, 100)) { + foreach (ref pkg; pkgs) { auto pkid = Package.getId (pkg); auto gcids = dcache.getGCIDsForPackage (pkid); if (gcids !is null) {