diff --git a/source/backends/debian/debpkg.d b/source/backends/debian/debpkg.d index 80902897..61da145a 100644 --- a/source/backends/debian/debpkg.d +++ b/source/backends/debian/debpkg.d @@ -35,7 +35,7 @@ import utils : isRemote, downloadFile; class DebPackage : Package { -private: +protected: string pkgname; string pkgver; string pkgarch; @@ -61,7 +61,7 @@ public: override @property string filename () const { if (debFname.isRemote) { - immutable path = buildPath (tmpDir, debFname.baseName); + immutable path = buildNormalizedPath (tmpDir, debFname.baseName); downloadFile (debFname, path); return path; } @@ -121,6 +121,19 @@ public: return pa; } + protected void extractPackage (const string dest = buildPath (tmpDir, name)) + { + import std.file : exists; + import std.regex : ctRegex; + + if (!dest.exists) + mkdirRecurse (dest); + + auto pa = openPayloadArchive (); + + pa.extractArchive (dest); + } + private auto openControlArchive () { auto ca = new ArchiveDecompressor (); diff --git a/source/backends/debian/debpkgindex.d b/source/backends/debian/debpkgindex.d index 393dacb7..b6c38546 100644 --- a/source/backends/debian/debpkgindex.d +++ b/source/backends/debian/debpkgindex.d @@ -39,7 +39,7 @@ import utils : escapeXml, getFileContents, isRemote; class DebianPackageIndex : PackageIndex { -private: +protected: string rootDir; Package[][string] pkgCache; bool[string] indexChanged; @@ -175,6 +175,11 @@ public: return downloadIfNecessary (rootDir, tmpDir, buildPath (path, "Packages.%s")); } + protected DebPackage newPackage (string name, string ver, string arch) + { + return new DebPackage (name, ver, arch); + } + private DebPackage[] loadPackages (string suite, string section, string arch) { auto indexFname = getIndexFile (suite, section, arch); @@ -195,7 +200,7 @@ public: if (!name) continue; - auto pkg = new DebPackage (name, ver, arch); + auto pkg = newPackage (name, ver, arch); pkg.filename = buildPath (rootDir, fname); pkg.maintainer = tagf.readField ("Maintainer"); diff --git a/source/backends/interfaces.d b/source/backends/interfaces.d index b039ddfe..fb35eae9 100644 --- a/source/backends/interfaces.d +++ b/source/backends/interfaces.d @@ -19,6 +19,9 @@ module backends.interfaces; +import appstream.Component; +import glib.KeyFile; + import std.string; import std.container; public import datastore; @@ -33,6 +36,7 @@ abstract class Package @property string ver () const @safe pure; @property string arch () const @safe pure; @property string maintainer () const; + @property Package[] otherPackages; /** * A associative array containing package descriptions. @@ -59,6 +63,11 @@ abstract class Package */ abstract const(ubyte)[] getFileData (string fname); + /** + * Retrieve backend-specific translations. + */ + string[string] getDesktopFileTranslations (KeyFile desktopFile, const string text) { return null; } + /** * Close the package. This function is called when we will * no longer request any file data from this package. diff --git a/source/backends/ubuntu/package.d b/source/backends/ubuntu/package.d new file mode 100644 index 00000000..97150cf9 --- /dev/null +++ b/source/backends/ubuntu/package.d @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2016 Matthias Klumpp + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module backends.ubuntu; + +public import backends.ubuntu.ubupkg; +public import backends.ubuntu.ubupkgindex; diff --git a/source/backends/ubuntu/ubupkg.d b/source/backends/ubuntu/ubupkg.d new file mode 100644 index 00000000..4742f281 --- /dev/null +++ b/source/backends/ubuntu/ubupkg.d @@ -0,0 +1,191 @@ +/* + * Copyright (C) 2016 Canonical Ltd + * Author: Iain Lane + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module backends.ubuntu.ubupkg; + +import appstream.Component; + +import backends.debian.debpkg; +import backends.interfaces; + +import glib.Internationalization; +import glib.KeyFile; + +import std.container : Array; + +import logging; + +import utils : DESKTOP_GROUP; + +extern (C) char *bindtextdomain (const char *domainname, const char *dirName) nothrow @nogc; + +class UbuntuPackage : DebPackage +{ + this (string pname, string pver, string parch, string globalTmpDir, ref Array!Package allPackages) + { + this.globalTmpDir = globalTmpDir; + this.langpackDir = buildPath (globalTmpDir, "langpacks"); + this.localeDir = buildPath (langpackDir, "locales"); + this.allPackages = allPackages; + super (pname, pver, parch); + } + + override string[string] getDesktopFileTranslations (KeyFile desktopFile, const string text) + { + string langpackdomain; + + try { + langpackdomain = desktopFile.getString (DESKTOP_GROUP, + "X-Ubuntu-Gettext-Domain"); + } catch { + try { + langpackdomain = desktopFile.getString (DESKTOP_GROUP, + "X-GNOME-Gettext-Domain"); + } catch { + return null; + } + } + + logDebug ("%s has langpack domain %s", name, langpackdomain); + + synchronized { + extractLangpacks (); + return getTranslations (langpackdomain, text); + } + } + +private: + string globalTmpDir; + string langpackDir; + string localeDir; + string[] langpackLocales; + Array!Package allPackages; + + private void extractLangpacks () + { + import std.algorithm : filter, map; + import std.array : appender, array, split; + import std.file : dirEntries, exists, SpanMode, readText; + import std.parallelism : parallel; + import std.path : baseName; + import std.process : Pid, spawnProcess, wait; + import std.string : splitLines, startsWith; + + auto path = buildPath (langpackDir, "usr", "share", "locale-langpack"); + + if (!langpackDir.exists) { + bool[string] extracted; + + langpackDir.mkdirRecurse (); + + foreach (pkg; allPackages) { + if (!pkg.name.startsWith ("language-pack") || pkg.name in extracted) + continue; + + UbuntuPackage upkg = to!UbuntuPackage (pkg); + + logDebug ("Extracting %s", pkg.name); + upkg.extractPackage (langpackDir); + + extracted[pkg.name] = true; + } + + auto supportedd = buildPath (langpackDir, "var", "lib", "locales", "supported.d"); + + localeDir.mkdirRecurse (); + + foreach (locale; parallel (supportedd.dirEntries (SpanMode.shallow), 5)) + { + foreach (ref line; locale.readText.splitLines) { + auto components = line.split (" "); + auto localecharset = components[0].split ("."); + + auto outdir = buildPath (localeDir, components[0]); + logDebug ("Generating locale in %s", outdir); + + auto pid = spawnProcess (["/usr/bin/localedef", + "--no-archive", + "-i", + localecharset[0], + "-c", + "-f", + components[1], + outdir]); + + scope (exit) wait (pid); + } + } + + } + + if (langpackLocales is null) + langpackLocales = localeDir.dirEntries (SpanMode.shallow) + .filter!(f => f.isDir) + .map!(f => f.name.baseName) + .array; + } + + string[string] getTranslations (const string domain, const string text) + { + import core.stdc.locale; + import core.stdc.string : strdup; + + import std.c.stdlib : getenv, setenv, unsetenv; + import std.string : fromStringz, toStringz; + + char *[char *] env; + + foreach (ref var; ["LC_ALL", "LANG", "LANGUAGE", "LC_MESSAGES"]) { + auto value = getenv (var.toStringz); + if (value !is null) { + env[var.toStringz] = getenv (var.toStringz).strdup; + unsetenv (var.toStringz); + } + } + + scope (exit) { + foreach (key, val; env) + setenv (key, val, false); + } + + setenv ("LOCPATH", localeDir.toStringz, true); + + auto initialLocale = setlocale (LC_ALL, ""); + scope(exit) setlocale (LC_ALL, initialLocale); + + auto dir = buildPath (langpackDir, + "usr", + "share", + "locale-langpack"); + + string[string] ret; + + foreach (ref locale; langpackLocales) { + setlocale (LC_ALL, locale.toStringz); + bindtextdomain (domain.toStringz, dir.toStringz); + auto translatedtext = Internationalization.dgettext (domain, text); + + if (text != translatedtext) + ret[locale] = translatedtext; + } + + return ret; + } +} diff --git a/source/backends/ubuntu/ubupkgindex.d b/source/backends/ubuntu/ubupkgindex.d new file mode 100644 index 00000000..a27169b7 --- /dev/null +++ b/source/backends/ubuntu/ubupkgindex.d @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2016 Canonical Ltd + * Author: Iain Lane + * + * Licensed under the GNU Lesser General Public License Version 3 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the license, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software. If not, see . + */ + +module backends.ubuntu.ubupkgindex; + +import backends.debian; +import backends.interfaces; +import backends.ubuntu.ubupkg; + +import std.container : Array, make; + +class UbuntuPackageIndex : DebianPackageIndex +{ + +public: + this (string dir) + { + /* + * UbuntuPackage needs to extract the langpacks, so we give it an array + * of all packages. We don't do this here, as you migh think makes + * sense, because it is a very expensive operation and we want to avoid + * doing it if it's not necessary (when no packages being processed are + * using langpacks). + */ + allPackages = make!(Array!Package); + super (dir); + } + + override DebPackage newPackage (string name, string ver, string arch) + { + return new UbuntuPackage (name, ver, arch, tmpDir, allPackages); + } + + override Package[] packagesFor (string suite, string section, string arch) + { + auto pkgs = super.packagesFor (suite, section, arch); + + allPackages ~= pkgs; + + return pkgs; + } +} + +private: + Array!Package allPackages; diff --git a/source/config.d b/source/config.d index 107e703a..5c728b9f 100644 --- a/source/config.d +++ b/source/config.d @@ -65,6 +65,7 @@ enum Backend Unknown, Dummy, Debian, + Ubuntu, Archlinux, RpmMd } @@ -276,6 +277,10 @@ class Config this.backend = Backend.Debian; this.metadataType = DataType.YAML; break; + case "ubuntu": + this.backend = Backend.Ubuntu; + this.metadataType = DataType.YAML; + break; case "arch": case "archlinux": this.backend = Backend.Archlinux; diff --git a/source/engine.d b/source/engine.d index b369267c..cdc8763d 100644 --- a/source/engine.d +++ b/source/engine.d @@ -41,6 +41,7 @@ import utils : copyDir, stringArrayToByteArray; import backends.interfaces; import backends.dummy; import backends.debian; +import backends.ubuntu; import backends.archlinux; import backends.rpmmd; @@ -72,6 +73,9 @@ public: case Backend.Debian: pkgIndex = new DebianPackageIndex (conf.archiveRoot); break; + case Backend.Ubuntu: + pkgIndex = new UbuntuPackageIndex (conf.archiveRoot); + break; case Backend.Archlinux: pkgIndex = new ArchPackageIndex (conf.archiveRoot); break; diff --git a/source/handlers/desktopparser.d b/source/handlers/desktopparser.d index 0e341593..7fe09f57 100644 --- a/source/handlers/desktopparser.d +++ b/source/handlers/desktopparser.d @@ -35,9 +35,6 @@ import result; import utils; import config : Config, FormatVersion; - -immutable DESKTOP_GROUP = "Desktop Entry"; - private string getLocaleFromKey (string key) { if (!localeValid (key)) @@ -204,11 +201,18 @@ Component parseDesktopFile (GeneratorResult gres, string fname, string data, boo if (key.startsWith ("Name")) { immutable val = getValue (df, key); checkDesktopString (key, val); - cpt.setName (val, locale); + /* run backend specific hooks */ + auto translations = gres.pkg.getDesktopFileTranslations (df, val); + translations[locale] = val; + foreach (key, value; translations) + cpt.setName (value, key); } else if (key.startsWith ("Comment")) { immutable val = getValue (df, key); checkDesktopString (key, val); - cpt.setSummary (val, locale); + auto translations = gres.pkg.getDesktopFileTranslations (df, val); + translations[locale] = val; + foreach (key, value; translations) + cpt.setSummary (value, key); } else if (key == "Categories") { auto value = getValue (df, key); auto cats = value.split (";"); @@ -219,13 +223,15 @@ Component parseDesktopFile (GeneratorResult gres, string fname, string data, boo foreach (ref c; cats) cpt.addCategory (c); } else if (key.startsWith ("Keywords")) { - auto value = getValue (df, key); - auto kws = value.split (";"); - kws = kws.stripRight (""); - if (kws.empty) - continue; - - cpt.setKeywords (kws, locale); + auto val = getValue (df, key); + auto translations = gres.pkg.getDesktopFileTranslations (df, val); + translations[locale] = val; + foreach (key, value; translations) { + auto kws = value.split (";").stripRight (""); + if (kws.empty) + continue; + cpt.setKeywords (kws, key); + } } else if (key == "MimeType") { auto value = getValue (df, key); immutable mts = value.split (";"); diff --git a/source/utils.d b/source/utils.d index 1fbbf06a..00b36311 100644 --- a/source/utils.d +++ b/source/utils.d @@ -32,6 +32,7 @@ static import std.file; import logging; +public immutable DESKTOP_GROUP = "Desktop Entry"; public immutable GENERIC_BUFFER_SIZE = 2048; diff --git a/source/zarchive.d b/source/zarchive.d index d1443c7e..d9c1e2fa 100644 --- a/source/zarchive.d +++ b/source/zarchive.d @@ -31,6 +31,14 @@ version (GNU) else import std.concurrency : Generator, yield; +version (unittest) { + version (GNU) { + extern(C) char *mkdtemp (char *) nothrow @nogc; + } else { + import core.sys.posix.stdlib; + } +} + import bindings.libarchive; private immutable DEFAULT_BLOCK_SIZE = 65536; @@ -247,6 +255,36 @@ public: return false; } + void extractArchive (const string dest) + in + { + import std.file : isDir; + assert (dest.isDir); + } + body + { + import std.path; + archive *ar; + archive_entry *en; + + ar = openArchive (); + scope(exit) archive_read_free (ar); + + while (archive_read_next_header (ar, &en) == ARCHIVE_OK) { + auto pathname = buildPath (dest, archive_entry_pathname (en).fromStringz); + /* at the moment we only handle directories and files */ + if (archive_entry_filetype (en) == AE_IFDIR) { + if (!pathname.exists) + pathname.mkdir; + continue; + } + + if (archive_entry_filetype (en) == AE_IFREG) { + this.extractEntryTo (ar, pathname); + } + } + } + const(ubyte)[] readData (string fname) { import core.sys.posix.sys.stat; @@ -553,4 +591,28 @@ unittest 0x00, 0x00, ]; assert (decompressData (emptyGz) == ""); + + writeln ("TEST: ", "Extracting a tarball"); + + import std.file : buildPath, tempDir; + import utils : getTestSamplesDir; + + auto archive = buildPath (getTestSamplesDir (), "test.tar.xz"); + assert (archive.exists); + auto ar = new ArchiveDecompressor (); + + auto tmpdir = buildPath (tempDir, "asgenXXXXXX"); + auto ctmpdir = new char[](tmpdir.length + 1); + ctmpdir[0 .. tmpdir.length] = tmpdir[]; + ctmpdir[$ - 1] = '\0'; + + tmpdir = to!string(mkdtemp (ctmpdir.ptr)); + scope(exit) rmdirRecurse (tmpdir); + + ar.open (archive); + ar.extractArchive (tmpdir); + + auto path = buildPath (tmpdir, "b", "a"); + assert (path.exists); + assert (path.readText.chomp == "hello"); } diff --git a/test/samples/test.tar.xz b/test/samples/test.tar.xz new file mode 100644 index 00000000..dd248b99 Binary files /dev/null and b/test/samples/test.tar.xz differ