Support manifest_hashes repository key for generation

11 files changed, 122 insertions, 21 deletions

diff --git a/NEWS b/NEWS
index 643068c6a..1a1f9ad0e 100644
--- a/NEWS
+++ b/NEWS
@@ -18,6 +18,9 @@ master:
 
     * The SHA-512 and Whirlpool functions are now supported in Manifest files.
 
+    * The hash functions used to generate Manifest files can now be controlled
+      by either the repository itself or user configuration.
+
 0.68.0:
     * Licence groups are now supported.
 
diff --git a/doc/configuration/repositories/e.html.part b/doc/configuration/repositories/e.html.part
index 93ede77b1..23c0fb8e7 100644
--- a/doc/configuration/repositories/e.html.part
+++ b/doc/configuration/repositories/e.html.part
@@ -94,6 +94,11 @@ for <code>e</code> format repositories:</p>
     <dd>Whether to use Manifest2. Valid values are <code>use</code>, <code>require</code> or <code>ignore</code>.
     Optional.</dd>
 
+    <dt><code>manifest_hashes</code></dt>
+    <dd>Space-separated list of hash functions to use when generating <code>Manifest</code> files. Supported values are
+    <code>MD5</code>, <code>RMD160</code>, <code>SHA1</code>, <code>SHA256</code>, <code>SHA512</code> and
+    <code>WHIRLPOOL</code>. Optional, usually set by the distribution or the repository's <code>metadata/layout.conf</code>.
+
     <dt><code>binary_destination</code></dt>
     <dd>If set to <code>true</code>, this repository is treated as a destination when creating binary packages.</dd>
 
diff --git a/paludis/distributions/exherbo/e.conf b/paludis/distributions/exherbo/e.conf
index abf28d5e6..bc63998f6 100644
--- a/paludis/distributions/exherbo/e.conf
+++ b/paludis/distributions/exherbo/e.conf
@@ -3,6 +3,7 @@ default_distdir = /var/cache/paludis/distfiles
 default_eapi_when_unknown = exheres-0
 default_eapi_when_unspecified = exheres-0
 default_layout = exheres
+default_manifest_hashes =
 default_names_cache = /var/cache/paludis/names
 default_profile_eapi = exheres-0
 default_profile_layout = exheres
diff --git a/paludis/distributions/gentoo/e.conf b/paludis/distributions/gentoo/e.conf
index ea4bf51da..673216ccf 100644
--- a/paludis/distributions/gentoo/e.conf
+++ b/paludis/distributions/gentoo/e.conf
@@ -3,6 +3,7 @@ default_distdir = distfiles
 default_eapi_when_unknown = 0
 default_eapi_when_unspecified = 0
 default_layout = traditional
+default_manifest_hashes = RMD160 SHA1 SHA256
 default_profile_layout = traditional
 default_names_cache =
 default_profile_eapi = 0
diff --git a/paludis/repositories/e/e_repository.cc b/paludis/repositories/e/e_repository.cc
index 78786a2f2..ad926686a 100644
--- a/paludis/repositories/e/e_repository.cc
+++ b/paludis/repositories/e/e_repository.cc
@@ -68,6 +68,7 @@
 #include <paludis/util/create_iterator-impl.hh>
 #include <paludis/util/deferred_construction_ptr.hh>
 #include <paludis/util/destringify.hh>
+#include <paludis/util/digest_registry.hh>
 #include <paludis/util/extract_host_from_url.hh>
 #include <paludis/util/fs_stat.hh>
 #include <paludis/util/fs_iterator.hh>
@@ -86,18 +87,16 @@
 #include <paludis/util/pimp-impl.hh>
 #include <paludis/util/process.hh>
 #include <paludis/util/return_literal_function.hh>
-#include <paludis/util/rmd160.hh>
 #include <paludis/util/safe_ifstream.hh>
 #include <paludis/util/safe_ofstream.hh>
 #include <paludis/util/sequence.hh>
 #include <paludis/util/set.hh>
-#include <paludis/util/sha1.hh>
-#include <paludis/util/sha256.hh>
 #include <paludis/util/stringify.hh>
 #include <paludis/util/strip.hh>
 #include <paludis/util/system.hh>
 #include <paludis/util/timestamp.hh>
 #include <paludis/util/tokeniser.hh>
+#include <paludis/util/upper_lower.hh>
 #include <paludis/util/wrapped_forward_iterator.hh>
 #include <paludis/util/wrapped_output_iterator.hh>
 
@@ -266,6 +265,7 @@ namespace paludis
         std::shared_ptr<const MetadataValueKey<std::string> > eapi_when_unspecified_key;
         std::shared_ptr<const MetadataValueKey<std::string> > profile_eapi_when_unspecified_key;
         std::shared_ptr<const MetadataValueKey<std::string> > use_manifest_key;
+        std::shared_ptr<const MetadataCollectionKey<Set<std::string> > > manifest_hashes_key;
         std::shared_ptr<const MetadataSectionKey> info_pkgs_key;
         std::shared_ptr<const MetadataCollectionKey<Set<std::string> > > info_vars_key;
         std::shared_ptr<const MetadataValueKey<std::string> > binary_destination_key;
@@ -345,6 +345,8 @@ namespace paludis
                     "profile_eapi_when_unspecified", "profile_eapi_when_unspecified", mkt_normal, params.profile_eapi_when_unspecified())),
         use_manifest_key(std::make_shared<LiteralMetadataValueKey<std::string> >(
                     "use_manifest", "use_manifest", mkt_normal, stringify(params.use_manifest()))),
+        manifest_hashes_key(std::make_shared<LiteralMetadataStringSetKey>(
+                    "manifest_hashes", "manifest_hashes", mkt_normal, params.manifest_hashes())),
         info_pkgs_key(layout->info_packages_files()->end() != std::find_if(layout->info_packages_files()->begin(),
                     layout->info_packages_files()->end(),
                     std::bind(std::mem_fn(&FSStat::is_regular_file_or_symlink_to_regular_file),
@@ -576,6 +578,7 @@ ERepository::_add_metadata_keys() const
     if (_imp->master_repositories_key)
         add_metadata_key(_imp->master_repositories_key);
     add_metadata_key(_imp->use_manifest_key);
+    add_metadata_key(_imp->manifest_hashes_key);
     if (_imp->info_pkgs_key)
         add_metadata_key(_imp->info_pkgs_key);
     if (_imp->info_vars_key)
@@ -1019,6 +1022,11 @@ ERepository::some_ids_might_not_be_masked() const
 void
 ERepository::make_manifest(const QualifiedPackageName & qpn)
 {
+    for (Set<std::string>::ConstIterator it(_imp->params.manifest_hashes()->begin()),
+             it_end(_imp->params.manifest_hashes()->end()); it_end != it; ++it)
+        if (! DigestRegistry::get_instance()->get(*it))
+            throw ERepositoryConfigurationError("Manifest hash function '" + *it + "' is not supported");
+
     FSPath package_dir = _imp->layout->package_directory(qpn);
 
     std::vector<std::pair<std::pair<std::string, std::string>, std::string> > lines;
@@ -1038,19 +1046,15 @@ ERepository::make_manifest(const QualifiedPackageName & qpn)
 
         SafeIFStream file_stream(file);
 
-        RMD160 rmd160sum(file_stream);
-        std::string line(file_type + " " + filename + " "
-            + stringify(file.stat().file_size()) + " RMD160 " + rmd160sum.hexsum());
-
-        file_stream.clear();
-        file_stream.seekg(0, std::ios::beg);
-        SHA1 sha1sum(file_stream);
-        line += " SHA1 " + sha1sum.hexsum();
+        std::string line(file_type + " " + filename + " " + stringify(file.stat().file_size()));
 
-        file_stream.clear();
-        file_stream.seekg(0, std::ios::beg);
-        SHA256 sha256sum(file_stream);
-        line += " SHA256 " + sha256sum.hexsum();
+        for (Set<std::string>::ConstIterator it(_imp->params.manifest_hashes()->begin()),
+                 it_end(_imp->params.manifest_hashes()->end()); it_end != it; ++it)
+        {
+            file_stream.clear();
+            file_stream.seekg(0, std::ios::beg);
+            line += " " + *it + " " + DigestRegistry::get_instance()->get(*it)(file_stream);
+        }
 
         lines.push_back(std::make_pair(std::make_pair(file_type, filename), line));
     }
@@ -1087,10 +1091,11 @@ ERepository::make_manifest(const QualifiedPackageName & qpn)
 
             MemoisedHashes * hashes = MemoisedHashes::get_instance();
 
-            std::string line("DIST " + f.basename() + " " + stringify(f_stat.file_size())
-                + " RMD160 " + hashes->get("RMD160", f, file_stream)
-                + " SHA1 " + hashes->get("SHA1", f, file_stream)
-                + " SHA256 " + hashes->get("SHA256", f, file_stream));
+            std::string line("DIST " + f.basename() + " " + stringify(f_stat.file_size()));
+
+            for (Set<std::string>::ConstIterator it(_imp->params.manifest_hashes()->begin()),
+                     it_end(_imp->params.manifest_hashes()->end()); it_end != it; ++it)
+                line += " " + *it + " " + hashes->get(*it, f, file_stream);
 
             lines.push_back(std::make_pair(std::make_pair("DIST", f.basename()), line));
         }
@@ -1455,6 +1460,19 @@ ERepository::repository_factory_create(
         use_manifest = destringify<UseManifest>(f("use_manifest"));
     }
 
+    std::shared_ptr<Set<std::string> > manifest_hashes_writable(std::make_shared<Set<std::string> >());
+    tokenise_whitespace(toupper(f("manifest_hashes")), manifest_hashes_writable->inserter());
+    if (manifest_hashes_writable->empty() && layout_conf)
+        // manifest-hashes with a hyphen, not an underscore (grrr)
+        tokenise_whitespace(toupper(layout_conf->get("manifest-hashes")), manifest_hashes_writable->inserter());
+    std::shared_ptr<const Set<std::string> > manifest_hashes;
+    if (! manifest_hashes_writable->empty())
+        manifest_hashes = manifest_hashes_writable;
+    else
+        manifest_hashes = EExtraDistributionData::get_instance()->data_from_distribution(
+                    *DistributionData::get_instance()->distribution_from_string(
+                        env->distribution()))->default_manifest_hashes();
+
     bool binary_destination(false);
 
     if (! f("binary_destination").empty())
@@ -1493,6 +1511,7 @@ ERepository::repository_factory_create(
                 n::ignore_deprecated_profiles() = ignore_deprecated_profiles,
                 n::layout() = layout,
                 n::location() = FSPath(location).realpath_if_exists(),
+                n::manifest_hashes() = manifest_hashes,
                 n::master_repositories() = master_repositories,
                 n::names_cache() = FSPath(names_cache).realpath_if_exists(),
                 n::newsdir() = FSPath(newsdir).realpath_if_exists(),
diff --git a/paludis/repositories/e/e_repository_TEST.cc b/paludis/repositories/e/e_repository_TEST.cc
index c9330f651..90a0e425a 100644
--- a/paludis/repositories/e/e_repository_TEST.cc
+++ b/paludis/repositories/e/e_repository_TEST.cc
@@ -663,6 +663,19 @@ TEST(ERepository, Manifest)
     EXPECT_EQ(contents("e_repository_TEST_dir/repo11/Manifest_correct"), contents("e_repository_TEST_dir/repo11/category/package/Manifest"));
 
     EXPECT_THROW(repo->make_manifest(QualifiedPackageName("category/package-b")), MissingDistfileError);
+
+    std::shared_ptr<Map<std::string, std::string> > keys2(std::make_shared<Map<std::string, std::string>>());
+    keys2->insert("format", "e");
+    keys2->insert("names_cache", "/var/empty");
+    keys2->insert("location", stringify(FSPath::cwd() / "e_repository_TEST_dir" / "repo11a"));
+    keys2->insert("profiles", stringify(FSPath::cwd() / "e_repository_TEST_dir" / "repo11a/profiles/profile"));
+    keys2->insert("builddir", stringify(FSPath::cwd() / "e_repository_TEST_dir" / "build"));
+    std::shared_ptr<ERepository> repo2(std::static_pointer_cast<ERepository>(ERepository::repository_factory_create(&env,
+                    std::bind(from_keys, keys2, std::placeholders::_1))));
+    env.add_repository(1, repo2);
+    repo2->make_manifest(QualifiedPackageName("category/package"));
+
+    EXPECT_EQ(contents("e_repository_TEST_dir/repo11a/Manifest_correct"), contents("e_repository_TEST_dir/repo11a/category/package/Manifest"));
 }
 
 TEST(ERepository, Fetch)
diff --git a/paludis/repositories/e/e_repository_TEST_setup.sh b/paludis/repositories/e/e_repository_TEST_setup.sh
index 7fb2d822a..7df730038 100755
--- a/paludis/repositories/e/e_repository_TEST_setup.sh
+++ b/paludis/repositories/e/e_repository_TEST_setup.sh
@@ -287,9 +287,10 @@ KEYWORDS="test"
 END
 cd ..
 
-mkdir -p repo11/{eclass,distfiles,profiles/profile} || exit 1
+mkdir -p repo11/{eclass,distfiles,metadata,profiles/profile} || exit 1
 mkdir -p repo11/category/package{,-b}/files || exit 1
 cd repo11 || exit 1
+echo "manifest-hashes = RMD160 SHA1 SHA256" >> metadata/layout.conf || exit 1
 echo "test-repo-11" >> profiles/repo_name || exit 1
 echo "category" >> profiles/categories || exit 1
 cat <<END > profiles/profile/make.defaults
@@ -347,6 +348,57 @@ DEPEND=""
 END
 cd ..
 
+mkdir -p repo11a/{eclass,distfiles,metadata,profiles/profile} || exit 1
+mkdir -p repo11a/category/package/files || exit 1
+cd repo11a || exit 1
+echo "manifest-hashes = SHA256 SHA512 WHIRLPOOL" > metadata/layout.conf || exit 1
+echo "test-repo-11a" >> profiles/repo_name || exit 1
+echo "category" >> profiles/categories || exit 1
+cat <<END > profiles/profile/make.defaults
+ARCH=test
+END
+cat <<END > category/package/package-1.ebuild || exit 1
+DESCRIPTION="The Description"
+HOMEPAGE="http://example.com/"
+SRC_URI="foo"
+SLOT="0"
+IUSE=""
+LICENSE="GPL-2"
+KEYWORDS="test"
+DEPEND=""
+END
+cat <<END > category/package/package-2.ebuild || exit 1
+DESCRIPTION="The Description"
+HOMEPAGE="http://example.com/"
+SRC_URI="bar"
+SLOT="0"
+IUSE=""
+LICENSE="GPL-2"
+KEYWORDS="test"
+DEPEND=""
+END
+cat <<END > category/package/ChangeLog || exit 1
+The times, they are-a changin'...
+END
+cat <<END > category/package/metadata.xml || exit
+This isn't valid xml & I don't care!
+END
+cat <<END > category/package/files/some.patch || exit 1
++ Manifest2
+END
+echo "something" > distfiles/foo || exit 1
+echo "for nothing" > distfiles/bar || exit 1
+cat <<END > Manifest_correct || exit 1
+AUX some.patch 12 SHA256 26955b4e2d4f60561b8002b72c34ce266f534a4f32f13a29aa33875d39d31cc4 SHA512 0b502928483c249c0cd142c771bcd0bc870ea422f1f5a5d69068a6e3d8417ea2d6660cf3fb68c8600cc33d91ca9cfc32762961696186801d4641829445ba1652 WHIRLPOOL fc49ed1a58063cfc598e20473e5e625582d8733b0306aabcecda642d985bc129bf083680403a4eec530708ac5f671610626ba6817c078cf43d4dcb47fd38ad36
+DIST bar 12 SHA256 27cd06afc317a809116e7730736663b9f09dd863fcc37b69d32d4f5eb58708b2 SHA512 eb482b4b17a46dbf7023c5caf76aed468ee32559b715df5a1539089c522913f612d7e780edca54546e8300813b41687550176be60899474ee8373183a19e98b0 WHIRLPOOL cbdc7a79ed68423b7d9fd25fc9f1c1cd01dfad53eca3d586083861918357d2081166b7939702eddf88a72ea8494694348b94a4df07775c2a7b1d1830470810ea
+DIST foo 10 SHA256 4bc453b53cb3d914b45f4b250294236adba2c0e09ff6f03793949e7e39fd4cc1 SHA512 4de57cffd81ee9dbf85831aff72914dc7ca7a7bef0466bfcda81ff3ef75d8a86d2f1c2f1bcb3afc130910384c700dd81d6a8eebdf81abfb5e61a1abcf70743fe WHIRLPOOL 02f9452201dba1f200fce2953487999b45eb85fbe1c4a518399b4640630b53b750d9bc171f37021be8e52ebc5a117394ec0435df2c2b85a1dc2e7a1e8cf75c7c
+EBUILD package-1.ebuild 134 SHA256 4d58e5622889397ff6a257d87652a8220585c4d97efbf0a42bf59b3f75d19e03 SHA512 c85f4539b0ba05b97c9cb75b226e211cf1ce1ac5619d4db344d1cad98235a9ee0839e7c0bd4e6ffbca96fea8f02fb76b4a63357e7c447abc678f0a3a9f609eef WHIRLPOOL 1c97d0f2f8c805de5238ab46a83ee05780ec6bb2cd0aaae5960b77a5ead328c570aeb5eec107a0247699a4b0fadbf9c833737a0b4a75905528c787f75a9607f9
+EBUILD package-2.ebuild 134 SHA256 3fb00f77d96c3e6576c2d424d31023958b507bdf20eb6555e89a135b37a54c07 SHA512 79b54d6aaa773540c77d943891957dbc060bfc714cc210343a7969eae96d64d43896f5221907a9dca5fe4de74e58dd6a36a943448e64c4ccd709c7391b3a7538 WHIRLPOOL 9212c730e541042be975e49f4ccd09816b9d02126c2cac3908296450d067bd0d2e39ec7120f1fbf75a9a32fb65ce8a2e860225da438f6cda89a7921ae56b8972
+MISC ChangeLog 34 SHA256 a8dfbbc187c93c0731fa9722aff87c437a4b9f59b1786d62651fb104b0c3ed97 SHA512 03d8f86f43de02a64a64f515a5a7ae97b544202ed60544b33814569d4b1502d1c9ce5f2e8e50a107aa2b08a0b127a815f90f11830197a0ecf34b67c019c0625f WHIRLPOOL 411b585cca9c3dcdf609967efc86f1996c2f1fd8a9d6b62549b9099611e312760c572ccbc6384bb0beab4a78b4b354dcf90284f4dab6640972fb38f8d944fcb5
+MISC metadata.xml 37 SHA256 ba3b181b832c002612fba7768c95e526e188658d8fc85b92c153940ad43169de SHA512 5120780bfcd7d0999bf108adfd02ddc1fa3d75666649fb64cf521fd1a94bb9653882a2a8f77d2825fe099e0a56b2858bd01a6439069585f2cc7d87f8ac5227d4 WHIRLPOOL 93bfb6ca3acc70b15dbd96bec3aa903219698bdc4aa141a2acca48d63c2928e237f4d6db524ff2ac98f4f10803726dc84975ad5ba8e41c32f6cbea901bb242aa
+END
+cd ..
+
 mkdir -p repo12/{profiles/profile,metadata} || exit 1
 cd repo12 || exit 1
 echo "test-repo-12" >> profiles/repo_name || exit 1
diff --git a/paludis/repositories/e/e_repository_params.hh b/paludis/repositories/e/e_repository_params.hh
index 7c3c368c4..fe9aacf59 100644
--- a/paludis/repositories/e/e_repository_params.hh
+++ b/paludis/repositories/e/e_repository_params.hh
@@ -23,6 +23,7 @@
 #include <paludis/util/fs_path.hh>
 #include <paludis/util/named_value.hh>
 #include <paludis/util/map-fwd.hh>
+#include <paludis/util/set-fwd.hh>
 #include <memory>
 
 /** \file
@@ -57,6 +58,7 @@ namespace paludis
         typedef Name<struct name_ignore_deprecated_profiles> ignore_deprecated_profiles;
         typedef Name<struct name_layout> layout;
         typedef Name<struct name_location> location;
+        typedef Name<struct name_manifest_hashes> manifest_hashes;
         typedef Name<struct name_master_repositories> master_repositories;
         typedef Name<struct name_names_cache> names_cache;
         typedef Name<struct name_newsdir> newsdir;
@@ -96,6 +98,7 @@ namespace paludis
             NamedValue<n::ignore_deprecated_profiles, bool> ignore_deprecated_profiles;
             NamedValue<n::layout, std::string> layout;
             NamedValue<n::location, FSPath> location;
+            NamedValue<n::manifest_hashes, std::shared_ptr<const Set<std::string> > > manifest_hashes;
             NamedValue<n::master_repositories, std::shared_ptr<const ERepositorySequence> > master_repositories;
             NamedValue<n::names_cache, FSPath> names_cache;
             NamedValue<n::newsdir, FSPath> newsdir;
diff --git a/paludis/repositories/e/extra_distribution_data.cc b/paludis/repositories/e/extra_distribution_data.cc
index 304d1e671..78e032559 100644
--- a/paludis/repositories/e/extra_distribution_data.cc
+++ b/paludis/repositories/e/extra_distribution_data.cc
@@ -21,6 +21,7 @@
 #include <paludis/util/make_named_values.hh>
 #include <paludis/util/set.hh>
 #include <paludis/util/tokeniser.hh>
+#include <paludis/util/upper_lower.hh>
 #include <paludis/util/wrapped_output_iterator.hh>
 #include <paludis/distribution-impl.hh>
 
@@ -55,6 +56,7 @@ namespace paludis
                             n::default_eapi_when_unknown() = k->get("default_eapi_when_unknown"),
                             n::default_eapi_when_unspecified() = k->get("default_eapi_when_unspecified"),
                             n::default_layout() = k->get("default_layout"),
+                            n::default_manifest_hashes() = make_set(toupper(k->get("default_manifest_hashes"))),
                             n::default_names_cache() = k->get("default_names_cache"),
                             n::default_profile_eapi() = k->get("default_profile_eapi"),
                             n::default_profile_layout() = k->get("default_profile_layout"),
diff --git a/paludis/repositories/e/extra_distribution_data.hh b/paludis/repositories/e/extra_distribution_data.hh
index c5cd4a740..68c366e5a 100644
--- a/paludis/repositories/e/extra_distribution_data.hh
+++ b/paludis/repositories/e/extra_distribution_data.hh
@@ -36,6 +36,7 @@ namespace paludis
         typedef Name<struct name_default_eapi_when_unknown> default_eapi_when_unknown;
         typedef Name<struct name_default_eapi_when_unspecified> default_eapi_when_unspecified;
         typedef Name<struct name_default_layout> default_layout;
+        typedef Name<struct name_default_manifest_hashes> default_manifest_hashes;
         typedef Name<struct name_default_names_cache> default_names_cache;
         typedef Name<struct name_default_profile_eapi> default_profile_eapi;
         typedef Name<struct name_default_profile_layout> default_profile_layout;
@@ -52,6 +53,7 @@ namespace paludis
             NamedValue<n::default_eapi_when_unknown, std::string> default_eapi_when_unknown;
             NamedValue<n::default_eapi_when_unspecified, std::string> default_eapi_when_unspecified;
             NamedValue<n::default_layout, std::string> default_layout;
+            NamedValue<n::default_manifest_hashes, std::shared_ptr<const Set<std::string> > > default_manifest_hashes;
             NamedValue<n::default_names_cache, std::string> default_names_cache;
             NamedValue<n::default_profile_eapi, std::string> default_profile_eapi;
             NamedValue<n::default_profile_layout, std::string> default_profile_layout;
diff --git a/vim/syntax/paludis-repositories-conf.vim b/vim/syntax/paludis-repositories-conf.vim
index 17a54e82d..5ff6ed86f 100644
--- a/vim/syntax/paludis-repositories-conf.vim
+++ b/vim/syntax/paludis-repositories-conf.vim
@@ -46,7 +46,7 @@ syn keyword PaludisRepositoriesConfKnownKey contained
             \ location distdir format builddir library sync root yaml_uri
             \ master_repository profiles pkgdir setsdir securitydir newsdir
             \ names_cache sync sync_options eclassdirs cache write_cache
-            \ importance layout use_manifest
+            \ importance layout use_manifest manifest_hashes
             \ binary_uri_prefix binary_keywords_filter binary_destination binary_distdir
             \ eapi_when_unspecified eapi_when_unknown profile_eapi_when_unspecified
             \ name handler config_template config_filename