aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAvatar Ciaran McCreesh <ciaran.mccreesh@googlemail.com> 2009-02-05 14:48:00 +0000
committerAvatar Ciaran McCreesh <ciaran.mccreesh@googlemail.com> 2009-02-05 14:48:00 +0000
commitb6feec685f0c4f0115c5a54fc9ce0687a013fbf9 (patch)
treee983d9456d1df940782394e535759d05322e20d6
parenta0a35b8d0dd4ca00c91350e3d3acf1bc6208cad1 (diff)
downloadpaludis-b6feec685f0c4f0115c5a54fc9ce0687a013fbf9.tar.gz
paludis-b6feec685f0c4f0115c5a54fc9ce0687a013fbf9.tar.xz
Kill grandmothers in parallel.
-rw-r--r--.gitignore1
-rw-r--r--paludis/util/files.m41
-rw-r--r--paludis/util/forward_parallel_for_each.hh96
-rw-r--r--paludis/util/forward_parallel_for_each_TEST.cc58
-rw-r--r--src/clients/inquisitio/do_search.cc11
5 files changed, 164 insertions, 3 deletions
diff --git a/.gitignore b/.gitignore
index 87c4850..5dda471 100644
--- a/.gitignore
+++ b/.gitignore
@@ -348,6 +348,7 @@ paludis-*.*.*.tar.bz2
/paludis/util/dir_iterator_TEST
/paludis/util/echo_functions.bash
/paludis/util/fast_unique_copy_TEST
+/paludis/util/forward_parallel_for_each_TEST
/paludis/util/fs_entry_TEST
/paludis/util/graph_TEST
/paludis/util/indirect_iterator_TEST
diff --git a/paludis/util/files.m4 b/paludis/util/files.m4
index 45151d5..7f330fe 100644
--- a/paludis/util/files.m4
+++ b/paludis/util/files.m4
@@ -25,6 +25,7 @@ add(`dir_iterator', `hh', `cc', `fwd', `se', `test', `tests
add(`exception', `hh', `cc')
add(`fast_unique_copy', `hh', `test')
add(`fd_output_stream', `hh')
+add(`forward_parallel_for_each', `hh', `test')
add(`fs_entry', `hh', `cc', `fwd', `test', `testscript')
add(`fd_holder', `hh')
add(`graph', `hh', `cc', `fwd', `impl', `test')
diff --git a/paludis/util/forward_parallel_for_each.hh b/paludis/util/forward_parallel_for_each.hh
new file mode 100644
index 0000000..2221a9b
--- /dev/null
+++ b/paludis/util/forward_parallel_for_each.hh
@@ -0,0 +1,96 @@
+/* vim: set sw=4 sts=4 et foldmethod=syntax : */
+
+/*
+ * Copyright (c) 2009 Ciaran McCreesh
+ *
+ * This file is part of the Paludis package manager. Paludis is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU General
+ * Public License version 2, as published by the Free Software Foundation.
+ *
+ * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef PALUDIS_GUARD_PALUDIS_UTIL_FORWARD_PARALLEL_FOR_EACH_HH
+#define PALUDIS_GUARD_PALUDIS_UTIL_FORWARD_PARALLEL_FOR_EACH_HH 1
+
+#include <paludis/util/mutex.hh>
+#include <paludis/util/thread_pool.hh>
+#include <tr1/functional>
+
+/** \file
+ * Declarations for the forward_parallel_for_each function.
+ *
+ * \ingroup g_threads
+ *
+ * \section Examples
+ *
+ * - None at this time.
+ */
+
+namespace paludis
+{
+ /**
+ * For forward_parallel_for_each.
+ *
+ * \see forward_parallel_for_each
+ * \since 0.36
+ * \ingroup g_threads
+ */
+ template <typename Iter_, typename Func_>
+ void forward_parallel_for_each_thread_func(Iter_ & cur, const Iter_ & end, Mutex & mutex, Func_ & func,
+ unsigned n_at_once)
+ {
+ while (true)
+ {
+ unsigned n_to_do(0);
+ Iter_ i(end);
+ {
+ Lock lock(mutex);
+ if (cur == end)
+ return;
+
+ i = cur;
+ while (n_to_do < n_at_once && cur != end)
+ {
+ ++cur;
+ ++n_to_do;
+ }
+ }
+
+ for (unsigned n(0) ; n < n_to_do ; ++n)
+ func(*i++);
+ }
+ }
+
+ /**
+ * Like std::for_each, but in parallel, and no return value.
+ *
+ * This works for forward iterators, but is only effective if each
+ * calculation is quite slow.
+ *
+ * \since 0.36
+ * \ingroup g_threads
+ */
+ template <typename Iter_, typename Func_>
+ void forward_parallel_for_each(Iter_ cur, const Iter_ & end, Func_ func, unsigned n_threads, unsigned n_at_once)
+ {
+ if (cur == end)
+ return;
+
+ Mutex mutex;
+ ThreadPool threads;
+
+ for (unsigned n(0) ; n != n_threads ; ++n)
+ threads.create_thread(std::tr1::bind(&forward_parallel_for_each_thread_func<Iter_, Func_>, std::tr1::ref(cur),
+ std::tr1::cref(end), std::tr1::ref(mutex), std::tr1::ref(func), n_at_once));
+ }
+}
+
+#endif
diff --git a/paludis/util/forward_parallel_for_each_TEST.cc b/paludis/util/forward_parallel_for_each_TEST.cc
new file mode 100644
index 0000000..53c8218
--- /dev/null
+++ b/paludis/util/forward_parallel_for_each_TEST.cc
@@ -0,0 +1,58 @@
+/* vim: set sw=4 sts=4 et foldmethod=syntax : */
+
+/*
+ * Copyright (c) 2009 Ciaran McCreesh
+ *
+ * This file is part of the Paludis package manager. Paludis is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU General
+ * Public License version 2, as published by the Free Software Foundation.
+ *
+ * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <paludis/util/forward_parallel_for_each.hh>
+#include <test/test_runner.hh>
+#include <test/test_framework.hh>
+#include <vector>
+#include <algorithm>
+
+using namespace test;
+using namespace paludis;
+
+namespace
+{
+ void inc(int & b) throw ()
+ {
+ ++b;
+ }
+}
+
+namespace test_cases
+{
+ struct ForwardParallelForEachTest : TestCase
+ {
+ ForwardParallelForEachTest() : TestCase("forward_parallel_for_each") { }
+
+ void run()
+ {
+ std::vector<int> t(1000, 0);
+
+ forward_parallel_for_each(t.begin(), t.end(), inc, 10, 1);
+ TEST_CHECK(1000 == std::count(t.begin(), t.end(), 1));
+
+ forward_parallel_for_each(t.begin(), t.end(), inc, 1, 100);
+ TEST_CHECK(1000 == std::count(t.begin(), t.end(), 2));
+
+ forward_parallel_for_each(t.begin(), t.end(), inc, 10, 10);
+ TEST_CHECK(1000 == std::count(t.begin(), t.end(), 3));
+ }
+ } test_forward_parallel_for_each;
+}
+
diff --git a/src/clients/inquisitio/do_search.cc b/src/clients/inquisitio/do_search.cc
index a76686d..64c5a40 100644
--- a/src/clients/inquisitio/do_search.cc
+++ b/src/clients/inquisitio/do_search.cc
@@ -1,7 +1,7 @@
/* vim: set sw=4 sts=4 et foldmethod=syntax : */
/*
- * Copyright (c) 2007, 2008 Ciaran McCreesh
+ * Copyright (c) 2007, 2008, 2009 Ciaran McCreesh
*
* This file is part of the Paludis package manager. Paludis is free software;
* you can redistribute it and/or modify it under the terms of the GNU General
@@ -34,6 +34,9 @@
#include <paludis/util/sequence.hh>
#include <paludis/util/make_shared_ptr.hh>
#include <paludis/util/create_iterator-impl.hh>
+#include <paludis/util/forward_parallel_for_each.hh>
+#include <paludis/util/system.hh>
+#include <paludis/util/destringify.hh>
#include <tr1/functional>
#include <list>
#include <set>
@@ -293,9 +296,11 @@ do_search(const Environment & env)
extractors
);
- std::for_each(ids.begin(), ids.end(), std::tr1::bind(&set_id, std::tr1::cref(env), std::tr1::cref(repos), _1, eligible, matches,
+ const unsigned n_threads(destringify<int>(getenv_with_default("INQUISITIO_THREADS", "5")));
+ forward_parallel_for_each(ids.begin(), ids.end(), std::tr1::bind(&set_id, std::tr1::cref(env), std::tr1::cref(repos), _1, eligible, matches,
CommandLine::get_instance()->a_all_versions.specified(),
- CommandLine::get_instance()->a_not.specified()));
+ CommandLine::get_instance()->a_not.specified()),
+ n_threads, 10);
bool any(false);
InquisitioQueryTask task(&env);