diff options
Diffstat (limited to 'paludis/ndbam.cc')
| -rw-r--r-- | paludis/ndbam.cc | 570 |
1 files changed, 570 insertions, 0 deletions
diff --git a/paludis/ndbam.cc b/paludis/ndbam.cc new file mode 100644 index 000000000..0187b0401 --- /dev/null +++ b/paludis/ndbam.cc @@ -0,0 +1,570 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2007, 2008 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <paludis/util/sequence-impl.hh> +#include <paludis/util/set.hh> +#include <paludis/util/stringify.hh> +#include <paludis/util/destringify.hh> +#include <paludis/util/dir_iterator.hh> +#include <paludis/util/tokeniser.hh> +#include <paludis/util/options.hh> +#include <paludis/util/log.hh> +#include <paludis/util/make_shared_ptr.hh> +#include <paludis/util/tr1_functional.hh> +#include <paludis/util/config_file.hh> +#include <paludis/ndbam.hh> +#include <paludis/hashed_containers.hh> +#include <paludis/package_id.hh> +#include <paludis/metadata_key.hh> +#include <paludis/name.hh> +#include <functional> +#include <vector> +#include <map> +#include <fstream> + +using namespace paludis; + +#include <paludis/ndbam-sr.cc> + +template class Sequence<NDBAMEntry>; + +namespace +{ + struct CategoryContents; + struct PackageContents; + struct CategoryNamesContainingPackageEntry; +} + +typedef MakeHashedMap<CategoryNamePart, tr1::shared_ptr<CategoryContents> >::Type CategoryContentsMap; +typedef MakeHashedMap<QualifiedPackageName, tr1::shared_ptr<PackageContents> >::Type PackageContentsMap; +typedef MakeHashedMap<PackageNamePart, tr1::shared_ptr<CategoryNamesContainingPackageEntry> >::Type CategoryNamesContainingPackage; + +namespace +{ + struct CategoryContents + { + Mutex mutex; + tr1::shared_ptr<QualifiedPackageNameSet> package_names; + PackageContentsMap package_contents_map; + }; + + struct PackageContents + { + Mutex mutex; + tr1::shared_ptr<NDBAMEntrySequence> entries; + }; + + struct CategoryNamesContainingPackageEntry + { + Mutex mutex; + tr1::shared_ptr<CategoryNamePartSet> category_names_containing_package; + }; +} + + +namespace paludis +{ + template <> + struct Implementation<NDBAM> + { + const FSEntry location; + + mutable Mutex category_names_mutex; + mutable tr1::shared_ptr<CategoryNamePartSet> category_names; + mutable CategoryContentsMap category_contents_map; + + mutable Mutex category_names_containing_package_mutex; + mutable CategoryNamesContainingPackage category_names_containing_package; + + Implementation(const FSEntry & l) : + location(l) + { + } + }; +} + +NDBAM::NDBAM(const FSEntry & l, + const tr1::function<bool (const std::string &)> & check_format, + const std::string & preferred_format) : + PrivateImplementationPattern<NDBAM>(new Implementation<NDBAM>(l)) +{ + Context c("When checking NDBAM layout at '" + stringify(l) + "':"); + if ((l / "ndbam.conf").exists()) + { + Context cc("When reading '" + stringify(l / "ndbam.conf") + "':"); + KeyValueConfigFile k(l / "ndbam.conf", KeyValueConfigFileOptions()); + if (k.get("ndbam_format") != "1") + throw ConfigurationError("Unsupported NDBAM format '" + k.get("ndbam_format") + "'"); + if (! check_format(k.get("repository_format"))) + throw ConfigurationError("Unsupported NDBAM repository format '" + k.get("ndbam_format") + "'"); + } + else if (DirIterator(l) != DirIterator()) + throw ConfigurationError("No NDBAM repository found at '" + stringify(l) + + "', and it is not an empty directory"); + else + { + Context cc("When creating skeleton NDBAM layout at '" + stringify(l) + "':"); + (l / "indices").mkdir(); + (l / "indices" / "categories").mkdir(); + (l / "indices" / "packages").mkdir(); + (l / "data").mkdir(); + std::ofstream n(stringify(l / "ndbam.conf").c_str()); + n << "ndbam_format = 1" << std::endl; + n << "repository_format = " << preferred_format << std::endl; + if (! n) + throw FSError("Could not write to '" + stringify(l / "ndbam.conf") + "'"); + } +} + +NDBAM::~NDBAM() +{ +} + +tr1::shared_ptr<const CategoryNamePartSet> +NDBAM::category_names() +{ + Lock l(_imp->category_names_mutex); + if (! _imp->category_names) + { + Context context("When loading category names for NDBAM at '" + stringify(_imp->location) + "':"); + _imp->category_names.reset(new CategoryNamePartSet); + for (DirIterator d(_imp->location / "indices" / "categories"), d_end ; + d != d_end ; ++d) + { + if (! d->is_directory_or_symlink_to_directory()) + continue; + if ('-' == d->basename().at(0)) + continue; + + try + { + CategoryNamePart c(d->basename()); + _imp->category_names->insert(c); + /* Inserting into category_contents_map might return false if + * we're partially populated. That's ok. */ + _imp->category_contents_map.insert(std::make_pair(c, make_shared_ptr(new CategoryContents))); + } + catch (const NameError & e) + { + Log::get_instance()->message(ll_warning, lc_context) << "Skipping directory '" << *d << "' due to exception '" + << e.message() << "' (" << e.what() << ")"; + } + } + } + + return _imp->category_names; +} + +tr1::shared_ptr<const QualifiedPackageNameSet> +NDBAM::package_names(const CategoryNamePart & c) +{ + if (! has_category_named(c)) + return make_shared_ptr(new QualifiedPackageNameSet); + + Lock l(_imp->category_names_mutex); + CategoryContentsMap::iterator cc_i(_imp->category_contents_map.find(c)); + if (_imp->category_contents_map.end() == cc_i) + throw InternalError(PALUDIS_HERE, "has_category_named(" + stringify(c) + ") but got category_contents_map end"); + CategoryContents & cc(*cc_i->second); + + l.acquire_then_release_old(cc.mutex); + + if (! cc.package_names) + { + Context context("When loading package names in '" + stringify(c) + "' for NDBAM at '" + stringify(_imp->location) + "':"); + cc.package_names.reset(new QualifiedPackageNameSet); + for (DirIterator d(_imp->location / "indices" / "categories" / stringify(c)), d_end ; + d != d_end ; ++d) + { + if (! d->is_directory_or_symlink_to_directory()) + continue; + if ('-' == d->basename().at(0)) + continue; + + try + { + QualifiedPackageName q(c + PackageNamePart(d->basename())); + cc.package_names->insert(q); + /* Inserting into package_contents_map might return false if + * we're partially populated. That's ok. */ + cc.package_contents_map.insert(std::make_pair(q, make_shared_ptr(new PackageContents))); + } + catch (const NameError & e) + { + Log::get_instance()->message(ll_warning, lc_context) << "Skipping directory '" << *d << "' due to exception '" + << e.message() << "' (" << e.what() << ")"; + } + } + } + return cc.package_names; +} + +bool +NDBAM::has_category_named(const CategoryNamePart & c) +{ + Lock l(_imp->category_names_mutex); + if (_imp->category_contents_map.end() != _imp->category_contents_map.find(c)) + return true; + + if (! _imp->category_names) + { + if (FSEntry(_imp->location / "indices" / "categories" / stringify(c)).is_directory_or_symlink_to_directory()) + { + _imp->category_contents_map.insert(std::make_pair(c, new CategoryContents)); + return true; + } + } + + return false; +} + +bool +NDBAM::has_package_named(const QualifiedPackageName & q) +{ + if (! has_category_named(q.category)) + return false; + + Lock l(_imp->category_names_mutex); + CategoryContentsMap::iterator cc_i(_imp->category_contents_map.find(q.category)); + if (_imp->category_contents_map.end() == cc_i) + throw InternalError(PALUDIS_HERE, "has_category_named(" + stringify(q.category) + ") but got category_contents_map end"); + + CategoryContents & cc(*cc_i->second); + l.acquire_then_release_old(cc.mutex); + + if (cc.package_contents_map.end() != cc.package_contents_map.find(q)) + return true; + + if (! cc.package_names) + { + if (FSEntry(_imp->location / "indices" / "categories" / stringify(q.category) / stringify(q.package)).is_directory_or_symlink_to_directory()) + { + cc.package_contents_map.insert(std::make_pair(q, new PackageContents)); + return true; + } + } + + return false; +} + +namespace +{ + template <typename T_> + T_ * desptr(const tr1::shared_ptr<T_> & p) + { + return p.get(); + } +} + +tr1::shared_ptr<NDBAMEntrySequence> +NDBAM::entries(const QualifiedPackageName & q) +{ + if (! has_package_named(q)) + return make_shared_ptr(new NDBAMEntrySequence); + + Lock l(_imp->category_names_mutex); + CategoryContentsMap::iterator cc_i(_imp->category_contents_map.find(q.category)); + if (_imp->category_contents_map.end() == cc_i) + throw InternalError(PALUDIS_HERE, "has_package_named(" + stringify(q) + ") but got category_contents_map end"); + CategoryContents & cc(*cc_i->second); + l.acquire_then_release_old(cc.mutex); + + PackageContentsMap::iterator pc_i(cc.package_contents_map.find(q)); + if (cc.package_contents_map.end() == pc_i) + throw InternalError(PALUDIS_HERE, "has_package_named(" + stringify(q) + ") but got package_contents_map end"); + PackageContents & pc(*pc_i->second); + l.acquire_then_release_old(pc.mutex); + + if (! pc.entries) + { + pc.entries.reset(new NDBAMEntrySequence); + Context context("When loading versions in '" + stringify(q) + "' for NDBAM at '" + stringify(_imp->location) + "':"); + pc.entries.reset(new NDBAMEntrySequence); + for (DirIterator d(_imp->location / "indices" / "categories" / stringify(q.category) / stringify(q.package)), d_end ; + d != d_end ; ++d) + { + if (! d->is_directory_or_symlink_to_directory()) + continue; + if ('-' == d->basename().at(0)) + continue; + + try + { + std::vector<std::string> tokens; + tokenise<delim_kind::AnyOfTag, delim_mode::DelimiterTag>(d->basename(), ":", "", std::back_inserter(tokens)); + if (tokens.size() < 3) + { + Log::get_instance()->message(ll_warning, lc_context) << "Not using '" << *d << + "', since it contains less than three ':'s"; + continue; + } + + VersionSpec v(tokens[0]); + SlotName s(tokens[1]); + std::string m(tokens[2]); + pc.entries->push_back(make_shared_ptr(new NDBAMEntry(NDBAMEntry::create() + .name(q) + .version(v) + .slot(s) + .fs_location(d->realpath()) + .package_id(tr1::shared_ptr<PackageID>()) + .magic(m) + .mutex(make_shared_ptr(new Mutex))))); + } + catch (const Exception & e) + { + Log::get_instance()->message(ll_warning, lc_context) << "Skipping directory '" << *d << "' due to exception '" + << e.message() << "' (" << e.what() << ")"; + } + } + + using namespace tr1::placeholders; + pc.entries->sort( + tr1::bind(std::less<VersionSpec>(), + tr1::bind<VersionSpec>(tr1::mem_fn(&NDBAMEntry::version), tr1::bind(&desptr<const NDBAMEntry>, _1)), + tr1::bind<VersionSpec>(tr1::mem_fn(&NDBAMEntry::version), tr1::bind(&desptr<const NDBAMEntry>, _2)) + )); + } + + return pc.entries; +} + +void +NDBAM::parse_contents(const PackageID & id, + const tr1::function<void (const FSEntry &, const std::string & md5, const time_t mtime)> & on_file, + const tr1::function<void (const FSEntry &)> & on_dir, + const tr1::function<void (const FSEntry &, const std::string & target, const time_t mtime)> & on_sym + ) const +{ + Context c("When fetching contents for '" + stringify(id) + "':"); + + if (! id.fs_location_key()) + throw InternalError(PALUDIS_HERE, "No id.fs_location_key"); + + FSEntry ff(id.fs_location_key()->value() / "contents"); + if (! ff.is_regular_file_or_symlink_to_regular_file()) + { + Log::get_instance()->message(ll_warning, lc_context) << "Contents file '" << ff << "' not a regular file, skipping"; + return; + } + + LineConfigFile f(ff, LineConfigFileOptions()); + for (LineConfigFile::ConstIterator line(f.begin()), line_end(f.end()) ; + line != line_end ; ++line) + { + std::map<std::string, std::string> tokens; + std::string::size_type p(0); + bool error(false); + while ((! error) && (p < line->length()) && (std::string::npos != p)) + { + std::string::size_type q(line->find('=', p)); + if (std::string::npos == q) + { + Log::get_instance()->message(ll_warning, lc_context) << "Malformed line '" << *line << "' in '" << ff << "'"; + error = true; + continue; + } + + std::string key(line->substr(p, q - p)), value; + p = q + 1; + while (p < line->length() && std::string::npos != p) + { + if ('\\' == (*line)[p]) + { + ++p; + if (p >= line->length() || std::string::npos == p) + { + Log::get_instance()->message(ll_warning, lc_context) << "Malformed line '" << *line << "' in '" << ff << "'"; + error = true; + break; + } + if ('n' == (*line)[p]) + value.append("\n"); + else + value.append(1, (*line)[p]); + ++p; + } + else if (' ' == (*line)[p]) + { + if (! tokens.insert(std::make_pair(key, value)).second) + Log::get_instance()->message(ll_warning, lc_context) << "Duplicate token '" << key << "' on line '" + << *line << "' in '" << ff << "'"; + key.clear(); + value.clear(); + ++p; + break; + } + else + { + value.append(1, (*line)[p]); + ++p; + } + } + + if ((! error) && (! key.empty())) + { + if (! tokens.insert(std::make_pair(key, value)).second) + Log::get_instance()->message(ll_warning, lc_context) << "Duplicate token '" << key << "' on line '" + << *line << "' in '" << ff << "'"; + } + } + + if (error) + continue; + + if (! tokens.count("type")) + { + Log::get_instance()->message(ll_warning, lc_context) << + "No key 'type' found on line '" << *line << "' in '" << ff << "'"; + continue; + } + std::string type(tokens.find("type")->second); + + if (! tokens.count("path")) + { + Log::get_instance()->message(ll_warning, lc_context) << + "No key 'path' found on line '" << *line << "' in '" << ff << "'"; + continue; + } + std::string path(tokens.find("path")->second); + + if ("file" == type) + { + if (! tokens.count("md5")) + { + Log::get_instance()->message(ll_warning, lc_context) << + "No key 'md5' found on sym line '" << *line << "' in '" << ff << "'"; + continue; + } + std::string md5(tokens.find("md5")->second); + + if (! tokens.count("mtime")) + { + Log::get_instance()->message(ll_warning, lc_context) << + "No key 'mtime' found on sym line '" << *line << "' in '" << ff << "'"; + continue; + } + time_t mtime(destringify<time_t>(tokens.find("mtime")->second)); + + on_file(path, md5, mtime); + } + else if ("dir" == type) + { + on_dir(path); + } + else if ("sym" == type) + { + if (! tokens.count("target")) + { + Log::get_instance()->message(ll_warning, lc_context) << + "No key 'target' found on sym line '" << *line << "' in '" << ff << "'"; + continue; + } + std::string target(tokens.find("target")->second); + + if (! tokens.count("mtime")) + { + Log::get_instance()->message(ll_warning, lc_context) << + "No key 'mtime' found on sym line '" << *line << "' in '" << ff << "'"; + continue; + } + time_t mtime(destringify<time_t>(tokens.find("mtime")->second)); + + on_sym(path, target, mtime); + } + else + Log::get_instance()->message(ll_warning, lc_context) << + "Unknown type '" << type << "' found on line '" << *line << "' in '" << ff << "'"; + } +} + +tr1::shared_ptr<const CategoryNamePartSet> +NDBAM::category_names_containing_package(const PackageNamePart & p) const +{ + Lock l(_imp->category_names_containing_package_mutex); + CategoryNamesContainingPackage::iterator cncp_i(_imp->category_names_containing_package.find(p)); + if (_imp->category_names_containing_package.end() == cncp_i) + cncp_i = _imp->category_names_containing_package.insert(std::make_pair(p, new CategoryNamesContainingPackageEntry)).first; + CategoryNamesContainingPackageEntry & cncp(*cncp_i->second); + + l.acquire_then_release_old(cncp.mutex); + if (! cncp.category_names_containing_package) + { + Context c("When finding category names containing package '" + stringify(p) + + "' in NDBAM at '" + stringify(_imp->location) + "':"); + + cncp.category_names_containing_package.reset(new CategoryNamePartSet); + FSEntry dd(_imp->location / "indices" / "packages" / stringify(p)); + if (dd.is_directory_or_symlink_to_directory()) + { + for (DirIterator d(dd), d_end ; + d != d_end ; ++d) + { + if (! d->is_directory_or_symlink_to_directory()) + continue; + if ('-' == d->basename().at(0)) + continue; + + try + { + cncp.category_names_containing_package->insert(CategoryNamePart(d->basename())); + } + catch (const Exception & e) + { + Log::get_instance()->message(ll_warning, lc_context) << "Skipping directory '" << *d << "' due to exception '" + << e.message() << "' (" << e.what() << ")"; + } + } + } + } + + return cncp.category_names_containing_package; +} + +void +NDBAM::deindex(const QualifiedPackageName & q) const +{ + Context context("When deindexing '" + stringify(q) + "' in NDBAM at '" + stringify(_imp->location) + "':"); + + FSEntry cp_index_sym(_imp->location / "indices" / "categories" / stringify(q.category) / stringify(q.package)); + cp_index_sym.unlink(); + + FSEntry pc_index_sym(_imp->location / "indices" / "packages" / stringify(q.package) / stringify(q.category)); + pc_index_sym.unlink(); +} + +void +NDBAM::index(const QualifiedPackageName & q, const std::string & d) const +{ + Context context("When indexing '" + stringify(q) + "' to '" + stringify(d) + + "' in NDBAM at '" + stringify(_imp->location) + "':"); + + FSEntry cp_index_sym(_imp->location / "indices" / "categories" / stringify(q.category)); + cp_index_sym.mkdir(); + cp_index_sym /= stringify(q.package); + if (! cp_index_sym.exists()) + cp_index_sym.symlink("../../../data/" + d); + + FSEntry pc_index_sym(_imp->location / "indices" / "packages" / stringify(q.package)); + pc_index_sym.mkdir(); + pc_index_sym /= stringify(q.category); + if (! pc_index_sym.exists()) + pc_index_sym.symlink("../../../data/" + d); +} + |