diff options
author | 2010-08-14 19:23:04 +0100 | |
---|---|---|
committer | 2010-08-14 19:23:04 +0100 | |
commit | 3c7d9f376cd53601aeb01d61c4085d1acf793f3b (patch) | |
tree | 900a6d63bb454f2bdafcea20537606c765655925 | |
parent | 239ef87963a413153d7e95c22999cec8e261ecfe (diff) | |
download | paludis-3c7d9f376cd53601aeb01d61c4085d1acf793f3b.tar.gz paludis-3c7d9f376cd53601aeb01d61c4085d1acf793f3b.tar.xz |
Optional indexing for cave search
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | Makefile.am | 2 | ||||
-rw-r--r-- | configure.ac | 30 | ||||
-rw-r--r-- | doc/clients/Makefile.am | 1 | ||||
-rw-r--r-- | src/clients/cave/Makefile.am | 18 | ||||
-rw-r--r-- | src/clients/cave/cmd_find_candidates.cc | 72 | ||||
-rw-r--r-- | src/clients/cave/cmd_find_candidates.hh | 3 | ||||
-rw-r--r-- | src/clients/cave/cmd_manage_search_index.cc | 311 | ||||
-rw-r--r-- | src/clients/cave/cmd_manage_search_index.hh | 43 | ||||
-rw-r--r-- | src/clients/cave/cmd_search.cc | 38 | ||||
-rw-r--r-- | src/clients/cave/cmd_search_cmdline.cc | 9 | ||||
-rw-r--r-- | src/clients/cave/cmd_search_cmdline.hh | 9 | ||||
-rw-r--r-- | src/clients/cave/command_factory.cc | 2 | ||||
-rw-r--r-- | src/clients/cave/search_extras.cc | 205 | ||||
-rw-r--r-- | src/clients/cave/search_extras.hh | 45 | ||||
-rw-r--r-- | src/clients/cave/search_extras_handle.cc | 87 | ||||
-rw-r--r-- | src/clients/cave/search_extras_handle.hh | 68 |
17 files changed, 920 insertions, 24 deletions
diff --git a/.gitignore b/.gitignore index efbf43848..e96286b3a 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,7 @@ paludis-*.*.*.tar.bz2 /doc/clients/cave-help.html /doc/clients/cave-import.html /doc/clients/cave-info.html +/doc/clients/cave-manage-search-index.html /doc/clients/cave-match.html /doc/clients/cave-owner.html /doc/clients/cave-perform.html diff --git a/Makefile.am b/Makefile.am index 98b2eb7f9..628098187 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,7 +14,7 @@ DISTCHECK_CONFIGURE_FLAGS = --enable-ruby --enable-ruby-doc --enable-vim \ --with-environments=default,portage \ --with-clients=default,accerso,appareo,cave,inquisitio,instruo \ --with-default-distribution=giant-space-monkey \ - --enable-htmltidy + --enable-htmltidy --enable-search-index automake-deps-built-hack.tmp : built-sources-subdirs touch $@ diff --git a/configure.ac b/configure.ac index d6b4ff01a..a09ac16b9 100644 --- a/configure.ac +++ b/configure.ac @@ -93,6 +93,7 @@ need_syck_check= need_gem_check= need_libarchive_check= need_resolver= +need_sqlite3_check= dnl {{{ we can use abi::__cxa_demangle AC_MSG_CHECKING([for abi::__cxa_demangle]) @@ -938,6 +939,22 @@ AC_SUBST([ENABLE_XML]) AM_CONDITIONAL([ENABLE_XML], test "x$ENABLE_XML" = "xyes") dnl }}} +dnl {{{ check for whether to build search index things +AC_MSG_CHECKING([whether to build search index support]) +AC_ARG_ENABLE([xml], + AS_HELP_STRING([--enable-search-index], [Enable search index (needs sqlite3)]), + [ENABLE_SEARCH_INDEX=$enableval + AC_MSG_RESULT([$enableval])], + [ENABLE_SEARCH_INDEX=no + AC_MSG_RESULT([no])]) +if test x"$ENABLE_SEARCH_INDEX" = "xyes" ; then + need_sqlite3_check=yes + AC_DEFINE([ENABLE_SEARCH_INDEX], [1], [Build search index support]) +fi +AC_SUBST([ENABLE_SEARCH_INDEX]) +AM_CONDITIONAL([ENABLE_SEARCH_INDEX], test "x$ENABLE_SEARCH_INDEX" = "xyes") +dnl }}} + dnl {{{ colourschemes AC_MSG_CHECKING([whether we like pink]) AC_ARG_ENABLE([pink], @@ -1548,6 +1565,19 @@ if test "x$need_gem_check" = "xyes" ; then fi dnl }}} +dnl {{{ sqlite3 check +NEED_SQLITE3=$need_sqlite3_check +if test "x$need_sqlite3_check" = "xyes" ; then + PKG_CHECK_MODULES(SQLITE3DEPS, [sqlite3], [], + [AC_MSG_ERROR([sqlite3 is required if --enable-search-index is used])]) + AC_SUBST(SQLITE3_CFLAGS) + AC_SUBST(SQLITE3_LIBS) +fi +AC_SUBST([NEED_SQLITE3]) +AM_CONDITIONAL([NEED_SQLITE3], test "x$NEED_SQLITE3" = "xyes") +dnl }}} + + dnl {{{ eselect or eclectic AC_MSG_CHECKING([for config framework]) AC_ARG_WITH([config-framework], diff --git a/doc/clients/Makefile.am b/doc/clients/Makefile.am index 30c7c63c4..2fbbd2523 100644 --- a/doc/clients/Makefile.am +++ b/doc/clients/Makefile.am @@ -37,6 +37,7 @@ CAVE_COMMANDS_HTML = \ cave-help.html \ cave-import.html \ cave-info.html \ + cave-manage-search-index.html \ cave-match.html \ cave-owner.html \ cave-perform.html \ diff --git a/src/clients/cave/Makefile.am b/src/clients/cave/Makefile.am index 3733c947c..244b24045 100644 --- a/src/clients/cave/Makefile.am +++ b/src/clients/cave/Makefile.am @@ -30,6 +30,7 @@ command_MANS = \ cave-help.1 \ cave-import.1 \ cave-info.1 \ + cave-manage-search-index.1 \ cave-match.1 \ cave-owner.1 \ cave-perform.1 \ @@ -114,6 +115,7 @@ libcave_a_SOURCES = \ cmd_help.cc cmd_help.hh \ cmd_import.cc cmd_import.hh \ cmd_info.cc cmd_info.hh \ + cmd_manage_search_index.cc cmd_manage_search_index.hh \ cmd_match.cc cmd_match.hh \ cmd_owner.cc cmd_owner.hh \ cmd_perform.cc cmd_perform.hh \ @@ -157,6 +159,7 @@ libcave_a_SOURCES = \ format_string.cc format_string.hh \ formats.cc formats.hh \ script_command.cc script_command.hh \ + search_extras_handle.cc search_extras_handle.hh \ select_format_for_spec.cc select_format_for_spec.hh \ owner_common.cc owner_common.hh \ resolve_common.cc resolve_common.hh \ @@ -177,13 +180,26 @@ DISTCLEANFILES = $(man_MANS) $(noinst_DATA) CLEANFILES += .keep -lib_LTLIBRARIES = libcavematchextras_@PALUDIS_PC_SLOT@.la +lib_LTLIBRARIES = + +lib_LTLIBRARIES += libcavematchextras_@PALUDIS_PC_SLOT@.la libcavematchextras_@PALUDIS_PC_SLOT@_la_SOURCES = match_extras.cc match_extras.hh libcavematchextras_@PALUDIS_PC_SLOT@_la_CXXFLAGS = $(AM_CXXFLAGS) @PCRECPPDEPS_CFLAGS@ libcavematchextras_@PALUDIS_PC_SLOT@_la_LIBADD = @PCRECPPDEPS_LIBS@ libcavematchextras_@PALUDIS_PC_SLOT@_la_LDFLAGS = -version-info @VERSION_LIB_CURRENT@:@VERSION_LIB_REVISION@:0 +if ENABLE_SEARCH_INDEX + +lib_LTLIBRARIES += libcavesearchextras_@PALUDIS_PC_SLOT@.la + +libcavesearchextras_@PALUDIS_PC_SLOT@_la_SOURCES = search_extras.cc search_extras.hh +libcavesearchextras_@PALUDIS_PC_SLOT@_la_CXXFLAGS = $(AM_CXXFLAGS) @SQLITE3DEPS_CFLAGS@ +libcavesearchextras_@PALUDIS_PC_SLOT@_la_LIBADD = @SQLITE3DEPS_LIBS@ +libcavesearchextras_@PALUDIS_PC_SLOT@_la_LDFLAGS = -version-info @VERSION_LIB_CURRENT@:@VERSION_LIB_REVISION@:0 + +endif + cavecommandsdir = $(libexecdir)/cave/commands cavecommands_DATA = .keep diff --git a/src/clients/cave/cmd_find_candidates.cc b/src/clients/cave/cmd_find_candidates.cc index 61041328d..737ba550e 100644 --- a/src/clients/cave/cmd_find_candidates.cc +++ b/src/clients/cave/cmd_find_candidates.cc @@ -18,18 +18,15 @@ */ #include "cmd_find_candidates.hh" +#include "search_extras_handle.hh" + #include <paludis/args/args.hh> #include <paludis/args/do_help.hh> + #include <paludis/name.hh> #include <paludis/environment.hh> #include <paludis/package_database.hh> #include <paludis/repository.hh> -#include <paludis/util/set.hh> -#include <paludis/util/wrapped_forward_iterator.hh> -#include <paludis/util/wrapped_output_iterator.hh> -#include <paludis/util/make_shared_copy.hh> -#include <paludis/util/indirect_iterator-impl.hh> -#include <paludis/util/simple_visitor_cast.hh> #include <paludis/generator.hh> #include <paludis/filtered_generator.hh> #include <paludis/filter.hh> @@ -38,9 +35,18 @@ #include <paludis/user_dep_spec.hh> #include <paludis/package_id.hh> #include <paludis/mask.hh> + +#include <paludis/util/set.hh> +#include <paludis/util/wrapped_forward_iterator.hh> +#include <paludis/util/wrapped_output_iterator.hh> +#include <paludis/util/make_shared_copy.hh> +#include <paludis/util/indirect_iterator-impl.hh> +#include <paludis/util/simple_visitor_cast.hh> + #include <cstdlib> #include <iostream> #include <algorithm> +#include <list> #include <set> #include "command_command_line.hh" @@ -74,10 +80,20 @@ namespace SearchCommandLineCandidateOptions search_options; SearchCommandLineMatchOptions match_options; + SearchCommandLineIndexOptions index_options; + + args::ArgsGroup g_hints; + args::StringArg a_name_description_substring_hint; FindCandidatesCommandLine() : search_options(this), - match_options(this) + match_options(this), + index_options(this), + g_hints(main_options_section(), "Hints", "Hints allow, but do not require, the search to return a " + "reduced set of results"), + a_name_description_substring_hint(&g_hints, "name-description-substring", '\0', + "Candidates whose name or description does not include the specified string as a substring " + "may be omitted") { } }; @@ -120,13 +136,11 @@ FindCandidatesCommand::run( return EXIT_SUCCESS; } - if (cmdline.begin_parameters() == cmdline.end_parameters()) - throw args::DoHelp("find-candidates requires at least one parameter"); + if (cmdline.begin_parameters() != cmdline.end_parameters()) + throw args::DoHelp("find-candidates takes no parameters"); - const std::shared_ptr<Set<std::string> > patterns(std::make_shared<Set<std::string>>()); - std::copy(cmdline.begin_parameters(), cmdline.end_parameters(), patterns->inserter()); - - run_hosted(env, cmdline.search_options, cmdline.match_options, patterns, &print_spec, &no_step); + run_hosted(env, cmdline.search_options, cmdline.match_options, cmdline.index_options, + cmdline.a_name_description_substring_hint.argument(), &print_spec, &no_step); return EXIT_SUCCESS; } @@ -140,11 +154,39 @@ FindCandidatesCommand::run_hosted( const std::shared_ptr<Environment> & env, const SearchCommandLineCandidateOptions & search_options, const SearchCommandLineMatchOptions &, - const std::shared_ptr<const Set<std::string> > &, + const SearchCommandLineIndexOptions & index_options, + const std::string & name_description_substring_hint, const std::function<void (const PackageDepSpec &)> & yield, const std::function<void (const std::string &)> & step) { - if (! search_options.a_matching.specified()) + if (index_options.a_index.specified()) + { + step("Searching index"); + + CaveSearchExtrasDB * db(SearchExtrasHandle::get_instance()->open_db_function(stringify(index_options.a_index.argument()).c_str())); + + std::list<std::string> specs; + + if (search_options.a_matching.specified()) + { + throw InternalError(PALUDIS_HERE, "not yet"); + } + else + SearchExtrasHandle::get_instance()->find_candidates_function(db, specs, + search_options.a_all_versions.specified(), + search_options.a_visible.specified(), + name_description_substring_hint); + + SearchExtrasHandle::get_instance()->cleanup_db_function(db); + + for (auto s(specs.begin()), s_end(specs.end()) ; + s != s_end ; ++s) + { + step("Checking indexed candidates"); + yield(parse_user_package_dep_spec(*s, env.get(), { })); + } + } + else if (! search_options.a_matching.specified()) { step("Searching repositories"); diff --git a/src/clients/cave/cmd_find_candidates.hh b/src/clients/cave/cmd_find_candidates.hh index b9e1e7f18..5929c703b 100644 --- a/src/clients/cave/cmd_find_candidates.hh +++ b/src/clients/cave/cmd_find_candidates.hh @@ -43,7 +43,8 @@ namespace paludis const std::shared_ptr<Environment> &, const SearchCommandLineCandidateOptions &, const SearchCommandLineMatchOptions &, - const std::shared_ptr<const Set<std::string> > &, + const SearchCommandLineIndexOptions &, + const std::string &, const std::function<void (const PackageDepSpec &)> &, const std::function<void (const std::string &)> &); diff --git a/src/clients/cave/cmd_manage_search_index.cc b/src/clients/cave/cmd_manage_search_index.cc new file mode 100644 index 000000000..8ea9ade01 --- /dev/null +++ b/src/clients/cave/cmd_manage_search_index.cc @@ -0,0 +1,311 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2010 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "cmd_manage_search_index.hh" +#include "search_extras.hh" +#include "search_extras_handle.hh" + +#include <paludis/args/args.hh> +#include <paludis/args/do_help.hh> + +#include <paludis/name.hh> +#include <paludis/environment.hh> +#include <paludis/package_database.hh> +#include <paludis/repository.hh> +#include <paludis/generator.hh> +#include <paludis/filtered_generator.hh> +#include <paludis/filter.hh> +#include <paludis/filter_handler.hh> +#include <paludis/selection.hh> +#include <paludis/user_dep_spec.hh> +#include <paludis/package_id.hh> +#include <paludis/mask.hh> +#include <paludis/metadata_key.hh> +#include <paludis/choice.hh> +#include <paludis/about.hh> +#include <paludis/notifier_callback.hh> + +#include <paludis/util/set.hh> +#include <paludis/util/wrapped_forward_iterator.hh> +#include <paludis/util/wrapped_output_iterator.hh> +#include <paludis/util/indirect_iterator-impl.hh> +#include <paludis/util/simple_visitor_cast.hh> +#include <paludis/util/iterator_funcs.hh> +#include <paludis/util/accept_visitor.hh> +#include <paludis/util/mutex.hh> + +#include <cstdlib> +#include <iostream> +#include <algorithm> +#include <list> +#include <map> + +#include "config.h" +#include "command_command_line.hh" + +using namespace paludis; +using namespace cave; +using std::cout; +using std::endl; + +namespace +{ + struct ManageStep + { + std::string stage; + }; + + struct DisplayCallback + { + mutable Mutex mutex; + mutable std::map<std::string, int> metadata; + mutable int steps; + int total; + mutable std::string stage; + mutable unsigned width; + + bool output; + + DisplayCallback() : + steps(0), + total(-1), + stage("Querying"), + width(stage.length() + 2), + output(::isatty(1)) + { + if (output) + cout << stage << ": " << std::flush; + } + + ~DisplayCallback() + { + if (output) + cout << endl << endl; + } + + void update() const + { + if (! output) + return; + + std::string s(stage + ": "); + s.append(stringify(steps)); + if (-1 != total) + s.append("/" + stringify(total)); + + if (! metadata.empty()) + { + std::multimap<int, std::string> biggest; + for (std::map<std::string, int>::const_iterator i(metadata.begin()), i_end(metadata.end()) ; + i != i_end ; ++i) + biggest.insert(std::make_pair(i->second, i->first)); + + int t(0), n(0); + std::string ss; + for (std::multimap<int, std::string>::const_reverse_iterator i(biggest.rbegin()), i_end(biggest.rend()) ; + i != i_end ; ++i) + { + ++n; + + if (n == 4) + ss.append(", ..."); + + if (n < 4) + { + if (! ss.empty()) + ss.append(", "); + + ss.append(stringify(i->first) + " " + i->second); + } + + t += i->first; + } + + if (! s.empty()) + s.append(", "); + s.append(stringify(t) + " metadata (" + ss + ") "); + } + + std::cout << std::string(width, '\010') << s; + + if (width > s.length()) + std::cout + << std::string(width - s.length(), ' ') + << std::string(width - s.length(), '\010'); + + width = s.length(); + std::cout << std::flush; + } + + void operator() (const NotifierCallbackEvent & event) const + { + event.accept(*this); + } + + void operator() (const ManageStep & s) const + { + if (! output) + return; + + Lock lock(mutex); + ++steps; + stage = s.stage; + update(); + } + + void visit(const NotifierCallbackGeneratingMetadataEvent & e) const + { + if (! output) + return; + + Lock lock(mutex); + ++metadata.insert(std::make_pair(stringify(e.repository()), 0)).first->second; + update(); + } + + void visit(const NotifierCallbackResolverStepEvent &) const + { + } + + void visit(const NotifierCallbackResolverStageEvent &) const + { + } + + void visit(const NotifierCallbackLinkageStepEvent &) const + { + } + }; + + struct ManageSearchIndexCommandLine : + CaveCommandCommandLine + { + args::ArgsGroup g_actions; + args::SwitchArg a_create; + + virtual std::string app_name() const + { + return "cave manage-search-index"; + } + + virtual std::string app_synopsis() const + { + return "Manages a search index for use by cave search."; + } + + virtual std::string app_description() const + { + return "Manages a search index for use by cave search. A search index is only valid until " + "a package is installed or uninstalled, or a sync is performed, or configuration is " + "changed."; + } + + ManageSearchIndexCommandLine() : + g_actions(main_options_section(), "Actions", "Specify which action to perform. Exactly one action must be specified."), + a_create(&g_actions, "create", 'c', "Create a new search index. The existing search index is removed if " + "it already exists", true) + { + add_usage_line("--create ~/cave-search-index"); + } + }; +} + +int +ManageSearchIndexCommand::run( + const std::shared_ptr<Environment> & env, + const std::shared_ptr<const Sequence<std::string > > & args + ) +{ + ManageSearchIndexCommandLine cmdline; + cmdline.run(args, "CAVE", "CAVE_MANAGE_SEARCH_INDEX_OPTIONS", "CAVE_MANAGE_SEARCH_INDEX_CMDLINE"); + + if (cmdline.a_help.specified()) + { + cout << cmdline; + return EXIT_SUCCESS; + } + + if (capped_distance(cmdline.begin_parameters(), cmdline.end_parameters(), 2) != 1) + throw args::DoHelp("manage-search-index requires exactly one parameter"); + + if (! cmdline.a_create.specified()) + throw args::DoHelp("exactly one action must be specified"); + + FSEntry index_file(*cmdline.begin_parameters()); + index_file.unlink(); + + { + DisplayCallback display_callback; + ScopedNotifierCallback display_callback_holder(env.get(), + NotifierCallbackFunction(std::cref(display_callback))); + + display_callback(ManageStep{"Creating DB"}); + CaveSearchExtrasDB * db(SearchExtrasHandle::get_instance()->create_db_function(stringify(index_file).c_str())); + + display_callback(ManageStep{"Querying"}); + auto ids((*env)[selection::AllVersionsSorted(generator::All())]); + display_callback.total = display_callback.steps + std::distance(ids->begin(), ids->end()) + 1; + + SearchExtrasHandle::get_instance()->starting_adds_function(db); + + bool is_best(false), had_best_visible(false); + std::string old_name; + for (auto i(ids->rbegin()), i_end(ids->rend()) ; + i != i_end ; ++i) + { + display_callback(ManageStep{"Writing"}); + + std::string name(stringify((*i)->name())), short_desc, long_desc; + if ((*i)->short_description_key()) + short_desc = (*i)->short_description_key()->value(); + if ((*i)->long_description_key()) + long_desc = (*i)->long_description_key()->value(); + + bool is_visible(! (*i)->masked()); + + if (name != old_name) + { + is_best = true; + had_best_visible = false; + old_name = name; + } + + bool is_best_visible(is_visible && ! had_best_visible); + if (is_best_visible) + had_best_visible = true; + + SearchExtrasHandle::get_instance()->add_candidate_function(db, stringify((*i)->uniquely_identifying_spec()), + is_visible, is_best, is_best_visible, name, short_desc, long_desc); + + is_best = false; + } + + display_callback(ManageStep{"Finalising"}); + SearchExtrasHandle::get_instance()->done_adds_function(db); + SearchExtrasHandle::get_instance()->cleanup_db_function(db); + } + + return EXIT_SUCCESS; +} + +std::shared_ptr<args::ArgsHandler> +ManageSearchIndexCommand::make_doc_cmdline() +{ + return std::make_shared<ManageSearchIndexCommandLine>(); +} + diff --git a/src/clients/cave/cmd_manage_search_index.hh b/src/clients/cave/cmd_manage_search_index.hh new file mode 100644 index 000000000..4c216dafd --- /dev/null +++ b/src/clients/cave/cmd_manage_search_index.hh @@ -0,0 +1,43 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2010 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef PALUDIS_GUARD_SRC_CLIENTS_CAVE_CMD_MANAGE_SEARCH_INDEX_HH +#define PALUDIS_GUARD_SRC_CLIENTS_CAVE_CMD_MANAGE_SEARCH_INDEX_HH 1 + +#include "command.hh" + +namespace paludis +{ + namespace cave + { + class PALUDIS_VISIBLE ManageSearchIndexCommand : + public Command + { + public: + int run( + const std::shared_ptr<Environment> &, + const std::shared_ptr<const Sequence<std::string > > & args + ); + + std::shared_ptr<args::ArgsHandler> make_doc_cmdline(); + }; + } +} + +#endif diff --git a/src/clients/cave/cmd_search.cc b/src/clients/cave/cmd_search.cc index c33a0fb40..3d5c8b1f1 100644 --- a/src/clients/cave/cmd_search.cc +++ b/src/clients/cave/cmd_search.cc @@ -22,8 +22,10 @@ #include "cmd_find_candidates.hh" #include "cmd_match.hh" #include "cmd_show.hh" + #include <paludis/args/args.hh> #include <paludis/args/do_help.hh> + #include <paludis/name.hh> #include <paludis/environment.hh> #include <paludis/package_database.hh> @@ -35,15 +37,18 @@ #include <paludis/selection.hh> #include <paludis/package_id.hh> #include <paludis/metadata_key.hh> +#include <paludis/action.hh> +#include <paludis/mask.hh> +#include <paludis/choice.hh> +#include <paludis/notifier_callback.hh> + #include <paludis/util/set.hh> #include <paludis/util/wrapped_forward_iterator.hh> #include <paludis/util/indirect_iterator-impl.hh> #include <paludis/util/wrapped_output_iterator.hh> #include <paludis/util/mutex.hh> -#include <paludis/action.hh> -#include <paludis/mask.hh> -#include <paludis/choice.hh> -#include <paludis/notifier_callback.hh> +#include <paludis/util/iterator_funcs.hh> + #include <cstdlib> #include <iostream> #include <algorithm> @@ -79,10 +84,12 @@ namespace SearchCommandLineCandidateOptions search_options; SearchCommandLineMatchOptions match_options; + SearchCommandLineIndexOptions index_options; SearchCommandLine() : search_options(this), - match_options(this) + match_options(this), + index_options(this) { add_usage_line("[ --name | --description | --key HOMEPAGE ] pattern ..."); add_note("'cave search' should only be used when a complex metadata search is required. To see " @@ -272,6 +279,25 @@ SearchCommand::run( const std::shared_ptr<Sequence<std::string> > show_args(std::make_shared<Sequence<std::string>>()); + std::string name_description_substring_hint; + do + { + /* cmd_match.cc has similar logic too */ + if (cmdline.match_options.a_key.specified()) + break; + + if ((cmdline.match_options.a_type.argument() != "text") && (cmdline.match_options.a_type.argument() != "exact")) + break; + + if (cmdline.match_options.a_not.specified()) + break; + + if ((! cmdline.match_options.a_and.specified()) && (1 != capped_distance(cmdline.begin_parameters(), cmdline.end_parameters(), 2))) + break; + + name_description_substring_hint = *cmdline.begin_parameters(); + } while (false); + { DisplayCallback display_callback; ScopedNotifierCallback display_callback_holder(env.get(), @@ -285,7 +311,7 @@ SearchCommand::run( std::shared_ptr<Set<QualifiedPackageName> > matches(std::make_shared<Set<QualifiedPackageName>>()); find_candidates_command.run_hosted(env, cmdline.search_options, cmdline.match_options, - patterns, std::bind( + cmdline.index_options, name_description_substring_hint, std::bind( &found_candidate, env, std::ref(match_command), std::cref(cmdline.match_options), std::placeholders::_1, patterns, std::function<void (const PackageDepSpec &)>(std::bind( &found_match, env, std::ref(matches), std::placeholders::_1 diff --git a/src/clients/cave/cmd_search_cmdline.cc b/src/clients/cave/cmd_search_cmdline.cc index 90652ab4a..6891766d1 100644 --- a/src/clients/cave/cmd_search_cmdline.cc +++ b/src/clients/cave/cmd_search_cmdline.cc @@ -60,3 +60,12 @@ SearchCommandLineMatchOptions::SearchCommandLineMatchOptions(args::ArgsHandler * { } +SearchCommandLineIndexOptions::SearchCommandLineIndexOptions(args::ArgsHandler * const h) : + ArgsSection(h, "Index Options"), + g_index_options(this, "Index Options", "Controls the use of an index. An index may be created using " + "cave manage-search-index. Note that strange errors or partial results may occur if the index " + "is not up to date."), + a_index(&g_index_options, "index", '\0', "Use the specified index file") +{ +} + diff --git a/src/clients/cave/cmd_search_cmdline.hh b/src/clients/cave/cmd_search_cmdline.hh index 87d3dbc47..8281e4797 100644 --- a/src/clients/cave/cmd_search_cmdline.hh +++ b/src/clients/cave/cmd_search_cmdline.hh @@ -57,6 +57,15 @@ namespace paludis args::ArgsGroup g_key_handling_options; args::SwitchArg a_enabled_only; }; + + struct SearchCommandLineIndexOptions : + args::ArgsSection + { + SearchCommandLineIndexOptions(args::ArgsHandler * const); + + args::ArgsGroup g_index_options; + args::StringArg a_index; + }; } } diff --git a/src/clients/cave/command_factory.cc b/src/clients/cave/command_factory.cc index ace093013..66d93555e 100644 --- a/src/clients/cave/command_factory.cc +++ b/src/clients/cave/command_factory.cc @@ -44,6 +44,7 @@ #include "cmd_help.hh" #include "cmd_import.hh" #include "cmd_info.hh" +#include "cmd_manage_search_index.hh" #include "cmd_match.hh" #include "cmd_owner.hh" #include "cmd_perform.hh" @@ -150,6 +151,7 @@ CommandFactory::CommandFactory() : _imp->handlers.insert(std::make_pair("help", std::bind(&make_command<HelpCommand>))); _imp->handlers.insert(std::make_pair("import", std::bind(&make_command<ImportCommand>))); _imp->handlers.insert(std::make_pair("info", std::bind(&make_command<InfoCommand>))); + _imp->handlers.insert(std::make_pair("manage-search-index", std::bind(&make_command<ManageSearchIndexCommand>))); _imp->handlers.insert(std::make_pair("match", std::bind(&make_command<MatchCommand>))); _imp->handlers.insert(std::make_pair("owner", std::bind(&make_command<OwnerCommand>))); _imp->handlers.insert(std::make_pair("perform", std::bind(&make_command<PerformCommand>))); diff --git a/src/clients/cave/search_extras.cc b/src/clients/cave/search_extras.cc new file mode 100644 index 000000000..96f3c43f7 --- /dev/null +++ b/src/clients/cave/search_extras.cc @@ -0,0 +1,205 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2010 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "search_extras.hh" +#include <paludis/util/exception.hh> +#include <paludis/util/stringify.hh> +#include <sqlite3.h> + +using namespace paludis; + +struct CaveSearchExtrasDB +{ + sqlite3 * db; + sqlite3_stmt * add_candidate; +}; + +extern "C" +CaveSearchExtrasDB * +cave_search_extras_create_db(const std::string & file) +{ + auto data(cave_search_extras_open_db(file)); + + if (SQLITE_OK != sqlite3_exec(data->db, "drop table if exists candidates", 0, 0, 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_exec drop candidates failed"); + + if (SQLITE_OK != sqlite3_exec(data->db, "create table candidates ( " + "spec text not null primary key, " + "is_visible int not null, " + "is_best int not_null, " + "is_best_visible int not_null, " + "name text not null, " + "short_desc text not null, " + "long_desc text not null" + ")", 0, 0, 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_exec create candidates failed"); + + if (SQLITE_OK != sqlite3_prepare_v2(data->db, "insert into candidates " + "( spec, is_visible, is_best, is_best_visible, name, short_desc, long_desc ) " + "values ( ?1, ?2, ?3, ?4, ?5, ?6, ?7 )", + -1, &data->add_candidate, 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_prepare_v2 insert into candidates failed"); + + return data; +} + +extern "C" CaveSearchExtrasDB * +cave_search_extras_open_db(const std::string & file) +{ + auto data(new CaveSearchExtrasDB); + if (SQLITE_OK != sqlite3_open(file.c_str(), &data->db)) + throw InternalError(PALUDIS_HERE, "sqlite3_open failed"); + + data->add_candidate = 0; + + return data; +} + +extern "C" +void +cave_search_extras_cleanup(CaveSearchExtrasDB * const data) +{ + if (data->add_candidate) + sqlite3_finalize(data->add_candidate); + + sqlite3_close(data->db); + delete data; +} + +extern "C" +void +cave_search_extras_add_candidate( + CaveSearchExtrasDB * const data, + const std::string & spec, + const bool visible, + const bool best, + const bool best_visible, + const std::string & name, + const std::string & short_desc, + const std::string & long_desc) +{ + if (SQLITE_OK != sqlite3_reset(data->add_candidate)) + throw InternalError(PALUDIS_HERE, "sqlite3_reset add candidate failed"); + if (SQLITE_OK != sqlite3_clear_bindings(data->add_candidate)) + throw InternalError(PALUDIS_HERE, "sqlite3_clear_bindings add candidate failed"); + + if (SQLITE_OK != sqlite3_bind_text(data->add_candidate, 1, spec.c_str(), spec.length(), SQLITE_TRANSIENT)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 1 failed"); + if (SQLITE_OK != sqlite3_bind_int(data->add_candidate, 2, visible ? 1 : 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 2 failed"); + if (SQLITE_OK != sqlite3_bind_int(data->add_candidate, 3, best ? 1 : 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 3 failed"); + if (SQLITE_OK != sqlite3_bind_int(data->add_candidate, 4, best_visible ? 1 : 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 4 failed"); + if (SQLITE_OK != sqlite3_bind_text(data->add_candidate, 5, name.c_str(), name.length(), SQLITE_TRANSIENT)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 5 failed"); + if (SQLITE_OK != sqlite3_bind_text(data->add_candidate, 6, short_desc.c_str(), short_desc.length(), SQLITE_TRANSIENT)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 6 failed"); + if (SQLITE_OK != sqlite3_bind_text(data->add_candidate, 7, long_desc.c_str(), long_desc.length(), SQLITE_TRANSIENT)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text add candidate 7 failed"); + + int code; + if (SQLITE_DONE != (code = sqlite3_step(data->add_candidate))) + throw InternalError(PALUDIS_HERE, "sqlite3_step failed: " + stringify(code)); +} + +extern "C" +void +cave_search_extras_starting_adds(CaveSearchExtrasDB * const data) +{ + if (SQLITE_OK != sqlite3_exec(data->db, "begin", 0, 0, 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_exec begin failed"); +} + +extern "C" +void +cave_search_extras_done_adds(CaveSearchExtrasDB * const data) +{ + if (SQLITE_OK != sqlite3_exec(data->db, "commit", 0, 0, 0)) + throw InternalError(PALUDIS_HERE, "sqlite3_exec commit failed"); +} + +extern "C" +void +cave_search_extras_find_candidates(CaveSearchExtrasDB * const data, + std::list<std::string> & out, + const bool all_versions, const bool visible, + const std::string & name_description_substring_hint) +{ + sqlite3_stmt * find_candidates; + + std::string s; + if (all_versions && visible) + s = "is_visible"; + else if (visible) + s = "is_best_visible"; + else if (all_versions) + s = "1"; + else + s = "is_best"; + + std::string h, p1; + if (! name_description_substring_hint.empty()) + { + h = " and ( name like ?1 escape '\\' or short_desc like ?1 escape '\\' or long_desc like ?1 escape '\\' )"; + + p1 = "%"; + for (auto i(name_description_substring_hint.begin()), i_end(name_description_substring_hint.end()) ; + i != i_end ; ++i) + switch (*i) + { + case '%': + case '_': + case '\\': + p1.append(1, '\\'); + /* fall through */ + default: + p1.append(1, *i); + } + p1.append("%"); + } + + int code; + if (SQLITE_OK != ((code = sqlite3_prepare_v2(data->db, ("select spec from candidates where " + s + " = 1" + h).c_str(), -1, &find_candidates, 0)))) + throw InternalError(PALUDIS_HERE, "sqlite3_prepare_v2 select from candidates failed:" + stringify(code)); + + if (! p1.empty()) + { + if (SQLITE_OK != sqlite3_bind_text(find_candidates, 1, p1.c_str(), p1.length(), SQLITE_TRANSIENT)) + throw InternalError(PALUDIS_HERE, "sqlite3_bind_text select from candidates 1 failed"); + } + + while (true) + { + code = sqlite3_step(find_candidates); + + if (code == SQLITE_DONE) + break; + else if (code == SQLITE_ROW) + { + std::string text(reinterpret_cast<const char *>(sqlite3_column_text(find_candidates, 0))); + out.push_back(text); + } + else + throw InternalError(PALUDIS_HERE, "sqlite3_step select from candidates failed:" + stringify(code)); + } + + sqlite3_finalize(find_candidates); +} + diff --git a/src/clients/cave/search_extras.hh b/src/clients/cave/search_extras.hh new file mode 100644 index 000000000..863ae8ccf --- /dev/null +++ b/src/clients/cave/search_extras.hh @@ -0,0 +1,45 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2010 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef PALUDIS_GUARD_SRC_CLIENTS_CAVE_SEARCH_EXTRAS_HH +#define PALUDIS_GUARD_SRC_CLIENTS_CAVE_SEARCH_EXTRAS_HH 1 + +#include <paludis/util/attributes.hh> +#include <string> +#include <list> + +struct CaveSearchExtrasDB; + +extern "C" CaveSearchExtrasDB * cave_search_extras_create_db(const std::string &) PALUDIS_VISIBLE PALUDIS_ATTRIBUTE((warn_unused_result)); + +extern "C" CaveSearchExtrasDB * cave_search_extras_open_db(const std::string &) PALUDIS_VISIBLE PALUDIS_ATTRIBUTE((warn_unused_result)); + +extern "C" void cave_search_extras_cleanup(CaveSearchExtrasDB * const) PALUDIS_VISIBLE; + +extern "C" void cave_search_extras_starting_adds(CaveSearchExtrasDB * const) PALUDIS_VISIBLE; + +extern "C" void cave_search_extras_add_candidate(CaveSearchExtrasDB * const, const std::string &, + const bool, const bool, const bool, const std::string &, const std::string &, const std::string &) PALUDIS_VISIBLE; + +extern "C" void cave_search_extras_done_adds(CaveSearchExtrasDB * const) PALUDIS_VISIBLE; + +extern "C" void cave_search_extras_find_candidates(CaveSearchExtrasDB * const, std::list<std::string> &, + const bool, const bool, const std::string &) PALUDIS_VISIBLE; + +#endif diff --git a/src/clients/cave/search_extras_handle.cc b/src/clients/cave/search_extras_handle.cc new file mode 100644 index 000000000..a6d337934 --- /dev/null +++ b/src/clients/cave/search_extras_handle.cc @@ -0,0 +1,87 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2010 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "search_extras_handle.hh" + +#include <paludis/util/singleton-impl.hh> + +#include <paludis/args/do_help.hh> + +#include <paludis/about.hh> + +#include <string.h> +#include <dlfcn.h> +#include <stdint.h> + +#define STUPID_CAST(type, val) reinterpret_cast<type>(reinterpret_cast<uintptr_t>(val)) + +using namespace paludis; +using namespace cave; + +SearchExtrasHandle::SearchExtrasHandle() : + handle(0), + create_db_function(0), + open_db_function(0), + cleanup_db_function(0), + starting_adds_function(0), + add_candidate_function(0), + done_adds_function(0), + find_candidates_function(0) +{ + handle = ::dlopen(("libcavesearchextras_" + stringify(PALUDIS_PC_SLOT) + ".so").c_str(), RTLD_NOW | RTLD_GLOBAL); + if (! handle) + throw args::DoHelp("Search index creation not available because dlopen said " + stringify(::dlerror())); + + create_db_function = STUPID_CAST(CreateDBFunction, ::dlsym(handle, "cave_search_extras_create_db")); + if (! create_db_function) + throw args::DoHelp("Search index creation not available because dlsym said " + stringify(::dlerror())); + + open_db_function = STUPID_CAST(CreateDBFunction, ::dlsym(handle, "cave_search_extras_open_db")); + if (! open_db_function) + throw args::DoHelp("Search index not available because dlsym said " + stringify(::dlerror())); + + cleanup_db_function = STUPID_CAST(CleanupDBFunction, ::dlsym(handle, "cave_search_extras_cleanup")); + if (! cleanup_db_function) + throw args::DoHelp("Search index not available because dlsym said " + stringify(::dlerror())); + + add_candidate_function = STUPID_CAST(AddCandidateFunction, ::dlsym(handle, "cave_search_extras_add_candidate")); + if (! add_candidate_function) + throw args::DoHelp("Search index creation not available because dlsym said " + stringify(::dlerror())); + + starting_adds_function = STUPID_CAST(StartingAddsFunction, ::dlsym(handle, "cave_search_extras_starting_adds")); + if (! starting_adds_function) + throw args::DoHelp("Search index creation not available because dlsym said " + stringify(::dlerror())); + + done_adds_function = STUPID_CAST(DoneAddsFunction, ::dlsym(handle, "cave_search_extras_done_adds")); + if (! done_adds_function) + throw args::DoHelp("Search index creation not available because dlsym said " + stringify(::dlerror())); + + find_candidates_function = STUPID_CAST(FindCandidatesFunction, ::dlsym(handle, "cave_search_extras_find_candidates")); + if (! find_candidates_function) + throw args::DoHelp("Search index not available because dlsym said " + stringify(::dlerror())); +} + +SearchExtrasHandle::~SearchExtrasHandle() +{ + if (handle) + ::dlclose(handle); +} + +template class Singleton<SearchExtrasHandle>; + diff --git a/src/clients/cave/search_extras_handle.hh b/src/clients/cave/search_extras_handle.hh new file mode 100644 index 000000000..dd85bd272 --- /dev/null +++ b/src/clients/cave/search_extras_handle.hh @@ -0,0 +1,68 @@ +/* vim: set sw=4 sts=4 et foldmethod=syntax : */ + +/* + * Copyright (c) 2010 Ciaran McCreesh + * + * This file is part of the Paludis package manager. Paludis is free software; + * you can redistribute it and/or modify it under the terms of the GNU General + * Public License version 2, as published by the Free Software Foundation. + * + * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef PALUDIS_GUARD_SRC_CLIENTS_CAVE_SEARCH_EXTRAS_HANDLE_HH +#define PALUDIS_GUARD_SRC_CLIENTS_CAVE_SEARCH_EXTRAS_HANDLE_HH 1 + +#include <paludis/util/singleton.hh> +#include <list> +#include <string> + +struct CaveSearchExtrasDB; + +namespace paludis +{ + namespace cave + { + struct SearchExtrasHandle : + Singleton<SearchExtrasHandle> + { + typedef CaveSearchExtrasDB * (* CreateDBFunction)(const std::string &); + typedef CaveSearchExtrasDB * (* OpenDBFunction)(const std::string &); + + typedef void (* CleanupDBFunction)(CaveSearchExtrasDB * const); + + typedef void (* AddCandidateFunction)(CaveSearchExtrasDB * const, const std::string &, + const bool, const bool, const bool, const std::string &, const std::string &, const std::string &); + typedef void (* StartingAddsFunction)(CaveSearchExtrasDB * const); + typedef void (* DoneAddsFunction)(CaveSearchExtrasDB * const); + + typedef void (* FindCandidatesFunction)(CaveSearchExtrasDB * const, std::list<std::string> &, + const bool, const bool, const std::string &); + + void * handle; + + CreateDBFunction create_db_function; + OpenDBFunction open_db_function; + + CleanupDBFunction cleanup_db_function; + + StartingAddsFunction starting_adds_function; + AddCandidateFunction add_candidate_function; + DoneAddsFunction done_adds_function; + + FindCandidatesFunction find_candidates_function; + + SearchExtrasHandle(); + ~SearchExtrasHandle(); + }; + } +} + +#endif |