From c34671dcc85e10cb2507fc5096d3f9a5ef593cd6 Mon Sep 17 00:00:00 2001 From: David Leverton Date: Sun, 20 Nov 2011 17:42:37 +0000 Subject: Support for generating thin Manifests --- NEWS | 3 + doc/configuration/repositories/e.html.part | 4 ++ paludis/distributions/exherbo/e.conf | 1 + paludis/distributions/gentoo/e.conf | 1 + paludis/repositories/e/e_repository.cc | 75 +++++++++++++++-------- paludis/repositories/e/e_repository_TEST.cc | 17 +++++ paludis/repositories/e/e_repository_TEST_setup.sh | 60 ++++++++++++++++++ paludis/repositories/e/e_repository_params.hh | 2 + paludis/repositories/e/extra_distribution_data.cc | 2 + paludis/repositories/e/extra_distribution_data.hh | 2 + vim/syntax/paludis-repositories-conf.vim | 2 +- 11 files changed, 143 insertions(+), 26 deletions(-) diff --git a/NEWS b/NEWS index 41d7987ad..c7370a7cb 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,9 @@ News for Paludis This file lists the major changes between versions. For a more detailed list of every change, see the Git log. +master: + * Generation of thin Manifests is now supported. + 0.70.1: * 'cave search' now has a '--case-sensitive' option. diff --git a/doc/configuration/repositories/e.html.part b/doc/configuration/repositories/e.html.part index ecfeff4d0..3c7a0829a 100644 --- a/doc/configuration/repositories/e.html.part +++ b/doc/configuration/repositories/e.html.part @@ -99,6 +99,10 @@ for e format repositories:

MD5, RMD160, SHA1, SHA256, SHA512 and WHIRLPOOL. Optional, usually set by the distribution or the repository's metadata/layout.conf. +
thin_manifests
+
If set to true, generated Manifest files will only contain DIST entries. + Optional, usually set by the distribution or the repository's metadata/layout.conf.
+
binary_destination
If set to true, this repository is treated as a destination when creating binary packages.
diff --git a/paludis/distributions/exherbo/e.conf b/paludis/distributions/exherbo/e.conf index bc63998f6..9f7bbc6a2 100644 --- a/paludis/distributions/exherbo/e.conf +++ b/paludis/distributions/exherbo/e.conf @@ -7,5 +7,6 @@ default_manifest_hashes = default_names_cache = /var/cache/paludis/names default_profile_eapi = exheres-0 default_profile_layout = exheres +default_thin_manifests = false default_write_cache = /var/cache/paludis/metadata news_directory = /var/lib/exherbo/news diff --git a/paludis/distributions/gentoo/e.conf b/paludis/distributions/gentoo/e.conf index 673216ccf..215977a15 100644 --- a/paludis/distributions/gentoo/e.conf +++ b/paludis/distributions/gentoo/e.conf @@ -7,5 +7,6 @@ default_manifest_hashes = RMD160 SHA1 SHA256 default_profile_layout = traditional default_names_cache = default_profile_eapi = 0 +default_thin_manifests = false default_write_cache = /var/empty news_directory = /var/lib/gentoo/news diff --git a/paludis/repositories/e/e_repository.cc b/paludis/repositories/e/e_repository.cc index 83b132efd..68380b58e 100644 --- a/paludis/repositories/e/e_repository.cc +++ b/paludis/repositories/e/e_repository.cc @@ -266,6 +266,7 @@ namespace paludis std::shared_ptr > profile_eapi_when_unspecified_key; std::shared_ptr > use_manifest_key; std::shared_ptr > > manifest_hashes_key; + std::shared_ptr > thin_manifests_key; std::shared_ptr info_pkgs_key; std::shared_ptr > > info_vars_key; std::shared_ptr > binary_destination_key; @@ -347,6 +348,8 @@ namespace paludis "use_manifest", "use_manifest", mkt_normal, stringify(params.use_manifest()))), manifest_hashes_key(std::make_shared( "manifest_hashes", "manifest_hashes", mkt_normal, params.manifest_hashes())), + thin_manifests_key(std::make_shared >( + "thin_manifests", "thin_manifests", mkt_normal, params.thin_manifests())), info_pkgs_key(layout->info_packages_files()->end() != std::find_if(layout->info_packages_files()->begin(), layout->info_packages_files()->end(), std::bind(std::mem_fn(&FSStat::is_regular_file_or_symlink_to_regular_file), @@ -579,6 +582,7 @@ ERepository::_add_metadata_keys() const add_metadata_key(_imp->master_repositories_key); add_metadata_key(_imp->use_manifest_key); add_metadata_key(_imp->manifest_hashes_key); + add_metadata_key(_imp->thin_manifests_key); if (_imp->info_pkgs_key) add_metadata_key(_imp->info_pkgs_key); if (_imp->info_vars_key) @@ -1032,31 +1036,34 @@ ERepository::make_manifest(const QualifiedPackageName & qpn) std::vector, std::string> > lines; auto files(_imp->layout->manifest_files(qpn, package_dir)); - for (auto f(files->begin()) ; f != files->end() ; ++f) + if (! _imp->params.thin_manifests()) { - FSPath file(f->first); - FSStat file_stat(file); - std::string filename = file.basename(); - std::string file_type(f->second); - - if ("AUX" == file_type) + for (auto f(files->begin()) ; f != files->end() ; ++f) { - filename = stringify(file).substr(stringify(package_dir / "files").length()+1); - } + FSPath file(f->first); + FSStat file_stat(file); + std::string filename = file.basename(); + std::string file_type(f->second); - SafeIFStream file_stream(file); + if ("AUX" == file_type) + { + filename = stringify(file).substr(stringify(package_dir / "files").length()+1); + } - std::string line(file_type + " " + filename + " " + stringify(file.stat().file_size())); + SafeIFStream file_stream(file); - for (Set::ConstIterator it(_imp->params.manifest_hashes()->begin()), - it_end(_imp->params.manifest_hashes()->end()); it_end != it; ++it) - { - file_stream.clear(); - file_stream.seekg(0, std::ios::beg); - line += " " + *it + " " + DigestRegistry::get_instance()->get(*it)(file_stream); - } + std::string line(file_type + " " + filename + " " + stringify(file.stat().file_size())); + + for (Set::ConstIterator it(_imp->params.manifest_hashes()->begin()), + it_end(_imp->params.manifest_hashes()->end()); it_end != it; ++it) + { + file_stream.clear(); + file_stream.seekg(0, std::ios::beg); + line += " " + *it + " " + DigestRegistry::get_instance()->get(*it)(file_stream); + } - lines.push_back(std::make_pair(std::make_pair(file_type, filename), line)); + lines.push_back(std::make_pair(std::make_pair(file_type, filename), line)); + } } std::shared_ptr versions; @@ -1104,12 +1111,16 @@ ERepository::make_manifest(const QualifiedPackageName & qpn) std::sort(lines.begin(), lines.end()); FSPath(package_dir / "Manifest").unlink(); - SafeOFStream manifest(FSPath(package_dir / "Manifest"), -1, true); - if (! manifest) - throw ERepositoryConfigurationError("Couldn't open Manifest for writing."); - for (auto it(lines.begin()), it_end(lines.end()); it_end != it; ++it) - manifest << it->second << std::endl; + if (! lines.empty()) + { + SafeOFStream manifest(FSPath(package_dir / "Manifest"), -1, true); + if (! manifest) + throw ERepositoryConfigurationError("Couldn't open Manifest for writing."); + + for (auto it(lines.begin()), it_end(lines.end()); it_end != it; ++it) + manifest << it->second << std::endl; + } } void @@ -1473,8 +1484,21 @@ ERepository::repository_factory_create( *DistributionData::get_instance()->distribution_from_string( env->distribution()))->default_manifest_hashes(); - bool binary_destination(false); + bool thin_manifests(false); + if (! f("thin_manifests").empty()) + { + Context item_context("When handling thin_manifests key:"); + thin_manifests = destringify(f("thin_manifests")); + } + else if (layout_conf && ! layout_conf->get("thin-manifests").empty()) + // match Portage parsing + thin_manifests = tolower(layout_conf->get("thin-manifests")) == "true"; + else + thin_manifests = EExtraDistributionData::get_instance()->data_from_distribution( + *DistributionData::get_instance()->distribution_from_string( + env->distribution()))->default_thin_manifests(); + bool binary_destination(false); if (! f("binary_destination").empty()) { Context item_context("When handling binary_destination key:"); @@ -1523,6 +1547,7 @@ ERepository::repository_factory_create( n::setsdir() = FSPath(setsdir).realpath_if_exists(), n::sync() = sync, n::sync_options() = sync_options, + n::thin_manifests() = thin_manifests, n::use_manifest() = use_manifest, n::write_bin_uri_prefix() = "", n::write_cache() = FSPath(write_cache).realpath_if_exists() diff --git a/paludis/repositories/e/e_repository_TEST.cc b/paludis/repositories/e/e_repository_TEST.cc index 90a0e425a..448fe14c3 100644 --- a/paludis/repositories/e/e_repository_TEST.cc +++ b/paludis/repositories/e/e_repository_TEST.cc @@ -676,6 +676,23 @@ TEST(ERepository, Manifest) repo2->make_manifest(QualifiedPackageName("category/package")); EXPECT_EQ(contents("e_repository_TEST_dir/repo11a/Manifest_correct"), contents("e_repository_TEST_dir/repo11a/category/package/Manifest")); + + std::shared_ptr > keys3(std::make_shared>()); + keys3->insert("format", "e"); + keys3->insert("names_cache", "/var/empty"); + keys3->insert("location", stringify(FSPath::cwd() / "e_repository_TEST_dir" / "repo11b")); + keys3->insert("profiles", stringify(FSPath::cwd() / "e_repository_TEST_dir" / "repo11b/profiles/profile")); + keys3->insert("builddir", stringify(FSPath::cwd() / "e_repository_TEST_dir" / "build")); + std::shared_ptr repo3(std::static_pointer_cast(ERepository::repository_factory_create(&env, + std::bind(from_keys, keys3, std::placeholders::_1)))); + env.add_repository(1, repo3); + repo3->make_manifest(QualifiedPackageName("category/package")); + + EXPECT_EQ(contents("e_repository_TEST_dir/repo11b/Manifest_correct"), contents("e_repository_TEST_dir/repo11b/category/package/Manifest")); + + EXPECT_TRUE(FSStat(FSPath("e_repository_TEST_dir/repo11b/category/package2/Manifest")).exists()); + repo3->make_manifest(QualifiedPackageName("category/package2")); + EXPECT_TRUE(! FSStat(FSPath("e_repository_TEST_dir/repo11b/category/package2/Manifest")).exists()); } TEST(ERepository, Fetch) diff --git a/paludis/repositories/e/e_repository_TEST_setup.sh b/paludis/repositories/e/e_repository_TEST_setup.sh index 7df730038..fc7eba6a6 100755 --- a/paludis/repositories/e/e_repository_TEST_setup.sh +++ b/paludis/repositories/e/e_repository_TEST_setup.sh @@ -291,6 +291,7 @@ mkdir -p repo11/{eclass,distfiles,metadata,profiles/profile} || exit 1 mkdir -p repo11/category/package{,-b}/files || exit 1 cd repo11 || exit 1 echo "manifest-hashes = RMD160 SHA1 SHA256" >> metadata/layout.conf || exit 1 +echo "thin-manifests = false" >> metadata/layout.conf || exit 1 echo "test-repo-11" >> profiles/repo_name || exit 1 echo "category" >> profiles/categories || exit 1 cat < profiles/profile/make.defaults @@ -352,6 +353,7 @@ mkdir -p repo11a/{eclass,distfiles,metadata,profiles/profile} || exit 1 mkdir -p repo11a/category/package/files || exit 1 cd repo11a || exit 1 echo "manifest-hashes = SHA256 SHA512 WHIRLPOOL" > metadata/layout.conf || exit 1 +echo "thin-manifests = false" >> metadata/layout.conf || exit 1 echo "test-repo-11a" >> profiles/repo_name || exit 1 echo "category" >> profiles/categories || exit 1 cat < profiles/profile/make.defaults @@ -399,6 +401,64 @@ MISC metadata.xml 37 SHA256 ba3b181b832c002612fba7768c95e526e188658d8fc85b92c153 END cd .. +mkdir -p repo11b/{eclass,distfiles,metadata,profiles/profile} || exit 1 +mkdir -p repo11b/category/{package,package2}/files || exit 1 +cd repo11b || exit 1 +echo "manifest-hashes = SHA256 SHA512 WHIRLPOOL" > metadata/layout.conf || exit 1 +echo "thin-manifests = true" >> metadata/layout.conf || exit 1 +echo "test-repo-11b" >> profiles/repo_name || exit 1 +echo "category" >> profiles/categories || exit 1 +cat < profiles/profile/make.defaults +ARCH=test +END +cat < category/package/package-1.ebuild || exit 1 +DESCRIPTION="The Description" +HOMEPAGE="http://example.com/" +SRC_URI="foo" +SLOT="0" +IUSE="" +LICENSE="GPL-2" +KEYWORDS="test" +DEPEND="" +END +cat < category/package/package-2.ebuild || exit 1 +DESCRIPTION="The Description" +HOMEPAGE="http://example.com/" +SRC_URI="bar" +SLOT="0" +IUSE="" +LICENSE="GPL-2" +KEYWORDS="test" +DEPEND="" +END +cat < category/package/ChangeLog || exit 1 +The times, they are-a changin'... +END +cat < category/package/metadata.xml || exit +This isn't valid xml & I don't care! +END +cat < category/package/files/some.patch || exit 1 ++ Manifest2 +END +cat < category/package2/package2-2.ebuild || exit 1 +DESCRIPTION="The Description" +HOMEPAGE="http://example.com/" +SRC_URI="" +SLOT="0" +IUSE="" +LICENSE="GPL-2" +KEYWORDS="test" +DEPEND="" +END +echo lalala > category/package2/Manifest || exit 1 +echo "something" > distfiles/foo || exit 1 +echo "for nothing" > distfiles/bar || exit 1 +cat < Manifest_correct || exit 1 +DIST bar 12 SHA256 27cd06afc317a809116e7730736663b9f09dd863fcc37b69d32d4f5eb58708b2 SHA512 eb482b4b17a46dbf7023c5caf76aed468ee32559b715df5a1539089c522913f612d7e780edca54546e8300813b41687550176be60899474ee8373183a19e98b0 WHIRLPOOL cbdc7a79ed68423b7d9fd25fc9f1c1cd01dfad53eca3d586083861918357d2081166b7939702eddf88a72ea8494694348b94a4df07775c2a7b1d1830470810ea +DIST foo 10 SHA256 4bc453b53cb3d914b45f4b250294236adba2c0e09ff6f03793949e7e39fd4cc1 SHA512 4de57cffd81ee9dbf85831aff72914dc7ca7a7bef0466bfcda81ff3ef75d8a86d2f1c2f1bcb3afc130910384c700dd81d6a8eebdf81abfb5e61a1abcf70743fe WHIRLPOOL 02f9452201dba1f200fce2953487999b45eb85fbe1c4a518399b4640630b53b750d9bc171f37021be8e52ebc5a117394ec0435df2c2b85a1dc2e7a1e8cf75c7c +END +cd .. + mkdir -p repo12/{profiles/profile,metadata} || exit 1 cd repo12 || exit 1 echo "test-repo-12" >> profiles/repo_name || exit 1 diff --git a/paludis/repositories/e/e_repository_params.hh b/paludis/repositories/e/e_repository_params.hh index fe9aacf59..f242a2550 100644 --- a/paludis/repositories/e/e_repository_params.hh +++ b/paludis/repositories/e/e_repository_params.hh @@ -70,6 +70,7 @@ namespace paludis typedef Name setsdir; typedef Name sync; typedef Name sync_options; + typedef Name thin_manifests; typedef Name use_manifest; typedef Name write_bin_uri_prefix; typedef Name write_cache; @@ -110,6 +111,7 @@ namespace paludis NamedValue setsdir; NamedValue > > sync; NamedValue > > sync_options; + NamedValue thin_manifests; NamedValue use_manifest; NamedValue write_bin_uri_prefix; NamedValue write_cache; diff --git a/paludis/repositories/e/extra_distribution_data.cc b/paludis/repositories/e/extra_distribution_data.cc index 78e032559..43149346a 100644 --- a/paludis/repositories/e/extra_distribution_data.cc +++ b/paludis/repositories/e/extra_distribution_data.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -60,6 +61,7 @@ namespace paludis n::default_names_cache() = k->get("default_names_cache"), n::default_profile_eapi() = k->get("default_profile_eapi"), n::default_profile_layout() = k->get("default_profile_layout"), + n::default_thin_manifests() = destringify(k->get("default_thin_manifests")), n::default_write_cache() = k->get("default_write_cache"), n::news_directory() = FSPath(k->get("news_directory")) )); diff --git a/paludis/repositories/e/extra_distribution_data.hh b/paludis/repositories/e/extra_distribution_data.hh index 68c366e5a..3a95d7bde 100644 --- a/paludis/repositories/e/extra_distribution_data.hh +++ b/paludis/repositories/e/extra_distribution_data.hh @@ -40,6 +40,7 @@ namespace paludis typedef Name default_names_cache; typedef Name default_profile_eapi; typedef Name default_profile_layout; + typedef Name default_thin_manifests; typedef Name default_write_cache; typedef Name news_directory; } @@ -57,6 +58,7 @@ namespace paludis NamedValue default_names_cache; NamedValue default_profile_eapi; NamedValue default_profile_layout; + NamedValue default_thin_manifests; NamedValue default_write_cache; NamedValue news_directory; }; diff --git a/vim/syntax/paludis-repositories-conf.vim b/vim/syntax/paludis-repositories-conf.vim index 5ff6ed86f..f3ee61184 100644 --- a/vim/syntax/paludis-repositories-conf.vim +++ b/vim/syntax/paludis-repositories-conf.vim @@ -46,7 +46,7 @@ syn keyword PaludisRepositoriesConfKnownKey contained \ location distdir format builddir library sync root yaml_uri \ master_repository profiles pkgdir setsdir securitydir newsdir \ names_cache sync sync_options eclassdirs cache write_cache - \ importance layout use_manifest manifest_hashes + \ importance layout use_manifest manifest_hashes thin_manifests \ binary_uri_prefix binary_keywords_filter binary_destination binary_distdir \ eapi_when_unspecified eapi_when_unknown profile_eapi_when_unspecified \ name handler config_template config_filename -- cgit v1.2.3