aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAvatar Ciaran McCreesh <ciaran.mccreesh@googlemail.com> 2009-12-06 09:05:00 +0000
committerAvatar Ciaran McCreesh <ciaran.mccreesh@googlemail.com> 2009-12-06 16:59:35 +0000
commitd7ce8b54b2e6b0acda640ba528f5153450a2c5ce (patch)
tree2419a04e3d7f564c12c11398e4b8d738140b5584
parent4e9142925c02ccbe933804542f84f8b8a3f028e1 (diff)
downloadpaludis-d7ce8b54b2e6b0acda640ba528f5153450a2c5ce.tar.gz
paludis-d7ce8b54b2e6b0acda640ba528f5153450a2c5ce.tar.xz
Parallelise serialisation for displaying resolutions
-rw-r--r--.gitignore1
-rw-r--r--paludis/resolver/resolver_TEST_serialisation.cc7
-rw-r--r--paludis/util/files.m41
-rw-r--r--paludis/util/string_list_stream-fwd.hh28
-rw-r--r--paludis/util/string_list_stream.cc146
-rw-r--r--paludis/util/string_list_stream.hh74
-rw-r--r--paludis/util/string_list_stream_TEST.cc91
-rw-r--r--src/clients/cave/cmd_resolve.cc23
8 files changed, 365 insertions, 6 deletions
diff --git a/.gitignore b/.gitignore
index 7f84d2f..07c8d69 100644
--- a/.gitignore
+++ b/.gitignore
@@ -411,6 +411,7 @@ paludis-*.*.*.tar.bz2
/paludis/util/sha256_TEST
/paludis/util/simple_parser_TEST
/paludis/util/stringify_TEST
+/paludis/util/string_list_stream_TEST
/paludis/util/strip_TEST
/paludis/util/system_TEST
/paludis/util/system_TEST_become_child
diff --git a/paludis/resolver/resolver_TEST_serialisation.cc b/paludis/resolver/resolver_TEST_serialisation.cc
index b33f380..44cc273 100644
--- a/paludis/resolver/resolver_TEST_serialisation.cc
+++ b/paludis/resolver/resolver_TEST_serialisation.cc
@@ -35,6 +35,7 @@
#include <paludis/util/map.hh>
#include <paludis/util/indirect_iterator-impl.hh>
#include <paludis/util/accept_visitor.hh>
+#include <paludis/util/string_list_stream.hh>
#include <paludis/user_dep_spec.hh>
#include <paludis/repository_factory.hh>
#include <paludis/package_database.hh>
@@ -76,11 +77,13 @@ namespace test_cases
std::tr1::shared_ptr<const ResolverLists> resolutions;
{
std::tr1::shared_ptr<const ResolverLists> orig_resolutions(get_resolutions("serialisation/target"));
- std::stringstream str;
+ StringListStream str;
Serialiser ser(str);
orig_resolutions->serialise(ser);
+ str.nothing_more_to_write();
- Deserialiser deser(&env, str.str());
+ std::string strstr((std::istreambuf_iterator<char>(str)), std::istreambuf_iterator<char>());
+ Deserialiser deser(&env, strstr);
Deserialisation desern("ResolverLists", deser);
resolutions = make_shared_ptr(new ResolverLists(ResolverLists::deserialise(desern)));
}
diff --git a/paludis/util/files.m4 b/paludis/util/files.m4
index 63d1c5c..710614c 100644
--- a/paludis/util/files.m4
+++ b/paludis/util/files.m4
@@ -71,6 +71,7 @@ add(`simple_parser', `hh', `cc', `test', `fwd')
add(`simple_visitor', `hh', `cc', `fwd', `impl')
add(`simple_visitor_cast', `hh', `cc', `fwd')
add(`stringify', `hh', `test')
+add(`string_list_stream', `hh', `cc', `fwd', `test')
add(`strip', `hh', `cc', `test')
add(`system', `hh', `cc', `test', `testscript')
add(`tail_output_stream', `hh', `cc', `fwd', `test')
diff --git a/paludis/util/string_list_stream-fwd.hh b/paludis/util/string_list_stream-fwd.hh
new file mode 100644
index 0000000..b5c8ff4
--- /dev/null
+++ b/paludis/util/string_list_stream-fwd.hh
@@ -0,0 +1,28 @@
+/* vim: set sw=4 sts=4 et foldmethod=syntax : */
+
+/*
+ * Copyright (c) 2009 Ciaran McCreesh
+ *
+ * This file is part of the Paludis package manager. Paludis is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU General
+ * Public License version 2, as published by the Free Software Foundation.
+ *
+ * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef PALUDIS_GUARD_PALUDIS_UTIL_STRING_LIST_STREAM_FWD_HH
+#define PALUDIS_GUARD_PALUDIS_UTIL_STRING_LIST_STREAM_FWD_HH 1
+
+namespace paludis
+{
+ struct StringListStream;
+}
+
+#endif
diff --git a/paludis/util/string_list_stream.cc b/paludis/util/string_list_stream.cc
new file mode 100644
index 0000000..b87e0dd
--- /dev/null
+++ b/paludis/util/string_list_stream.cc
@@ -0,0 +1,146 @@
+/* vim: set sw=4 sts=4 et foldmethod=syntax : */
+
+/*
+ * Copyright (c) 2009 Ciaran McCreesh
+ *
+ * This file is part of the Paludis package manager. Paludis is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU General
+ * Public License version 2, as published by the Free Software Foundation.
+ *
+ * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <paludis/util/string_list_stream.hh>
+#include <paludis/util/private_implementation_pattern-impl.hh>
+#include <paludis/util/mutex.hh>
+#include <paludis/util/condition_variable.hh>
+#include <list>
+#include <string>
+
+using namespace paludis;
+
+namespace paludis
+{
+ template <>
+ struct Implementation<StringListStreamBuf>
+ {
+ Mutex mutex;
+ ConditionVariable condition;
+
+ std::string active_string;
+ std::list<std::string> future_strings;
+
+ bool nothing_more_to_write;
+
+ Implementation() :
+ nothing_more_to_write(false)
+ {
+ }
+ };
+}
+
+StringListStreamBuf::StringListStreamBuf() :
+ PrivateImplementationPattern<StringListStreamBuf>(new Implementation<StringListStreamBuf>())
+{
+ setg(0, 0, 0);
+}
+
+StringListStreamBuf::~StringListStreamBuf()
+{
+}
+
+StringListStreamBuf::int_type
+StringListStreamBuf::overflow(int_type c)
+{
+ Lock lock(_imp->mutex);
+ _imp->condition.signal();
+
+ if (c != traits_type::eof())
+ {
+ if ((_imp->future_strings.empty()) || (_imp->future_strings.back().length() > 1024))
+ _imp->future_strings.push_back(std::string(1, c));
+ else
+ _imp->future_strings.back().append(std::string(1, c));
+ }
+
+ return c;
+}
+
+std::streamsize
+StringListStreamBuf::xsputn(const char * s, std::streamsize num)
+{
+ Lock lock(_imp->mutex);
+ _imp->condition.signal();
+
+ if ((_imp->future_strings.empty()) || (_imp->future_strings.back().length() + num > 1024))
+ _imp->future_strings.push_back(std::string(s, num));
+ else
+ _imp->future_strings.back().append(std::string(s, num));
+ return num;
+}
+
+void
+StringListStreamBuf::nothing_more_to_write()
+{
+ Lock lock(_imp->mutex);
+ _imp->nothing_more_to_write = true;
+ _imp->condition.signal();
+}
+
+StringListStreamBuf::int_type
+StringListStreamBuf::underflow()
+{
+ Lock lock(_imp->mutex);
+
+ if (gptr() < egptr())
+ return traits_type::to_int_type(*gptr());
+
+ while (_imp->future_strings.empty())
+ {
+ if (_imp->nothing_more_to_write)
+ return traits_type::eof();
+
+ _imp->condition.wait(_imp->mutex);
+ }
+
+ _imp->active_string = *_imp->future_strings.begin();
+ _imp->future_strings.erase(_imp->future_strings.begin());
+ setg(&_imp->active_string[0], &_imp->active_string[0],
+ &_imp->active_string[0] + _imp->active_string.length());
+
+ return traits_type::to_int_type(*gptr());
+}
+
+StringListStreamBase::StringListStreamBase()
+{
+}
+
+StringListStreamBase::~StringListStreamBase()
+{
+}
+
+StringListStream::StringListStream() :
+ std::istream(&buf),
+ std::ostream(&buf)
+{
+}
+
+StringListStream::~StringListStream()
+{
+}
+
+void
+StringListStream::nothing_more_to_write()
+{
+ flush();
+ buf.nothing_more_to_write();
+}
+
+template class PrivateImplementationPattern<StringListStreamBuf>;
diff --git a/paludis/util/string_list_stream.hh b/paludis/util/string_list_stream.hh
new file mode 100644
index 0000000..d7c40b7
--- /dev/null
+++ b/paludis/util/string_list_stream.hh
@@ -0,0 +1,74 @@
+/* vim: set sw=4 sts=4 et foldmethod=syntax : */
+
+/*
+ * Copyright (c) 2009 Ciaran McCreesh
+ *
+ * This file is part of the Paludis package manager. Paludis is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU General
+ * Public License version 2, as published by the Free Software Foundation.
+ *
+ * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef PALUDIS_GUARD_PALUDIS_UTIL_STRING_LIST_STREAM_HH
+#define PALUDIS_GUARD_PALUDIS_UTIL_STRING_LIST_STREAM_HH 1
+
+#include <paludis/util/string_list_stream-fwd.hh>
+#include <paludis/util/private_implementation_pattern.hh>
+#include <istream>
+#include <ostream>
+
+namespace paludis
+{
+ class PALUDIS_VISIBLE StringListStreamBuf :
+ private PrivateImplementationPattern<StringListStreamBuf>,
+ public std::streambuf
+ {
+ protected:
+ virtual int_type underflow();
+
+ virtual int_type overflow(int_type c);
+ virtual std::streamsize xsputn(const char * s, std::streamsize num);
+
+ public:
+ StringListStreamBuf();
+ ~StringListStreamBuf();
+
+ void nothing_more_to_write();
+ };
+
+ class PALUDIS_VISIBLE StringListStreamBase
+ {
+ protected:
+ StringListStreamBuf buf;
+
+ public:
+ StringListStreamBase();
+ ~StringListStreamBase();
+ };
+
+ class PALUDIS_VISIBLE StringListStream :
+ protected StringListStreamBase,
+ public std::istream,
+ public std::ostream
+ {
+ public:
+ StringListStream();
+ ~StringListStream();
+
+ void nothing_more_to_write();
+ };
+
+#ifdef PALUDIS_HAVE_EXTERN_TEMPLATE
+ extern template class PrivateImplementationPattern<StringListStreamBuf>;
+#endif
+}
+
+#endif
diff --git a/paludis/util/string_list_stream_TEST.cc b/paludis/util/string_list_stream_TEST.cc
new file mode 100644
index 0000000..0e8ebff
--- /dev/null
+++ b/paludis/util/string_list_stream_TEST.cc
@@ -0,0 +1,91 @@
+/* vim: set sw=4 sts=4 et foldmethod=syntax : */
+
+/*
+ * Copyright (c) 2009 Ciaran McCreesh
+ *
+ * This file is part of the Paludis package manager. Paludis is free software;
+ * you can redistribute it and/or modify it under the terms of the GNU General
+ * Public License version 2, as published by the Free Software Foundation.
+ *
+ * Paludis is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <paludis/util/string_list_stream.hh>
+#include <paludis/util/thread.hh>
+#include <test/test_runner.hh>
+#include <test/test_framework.hh>
+#include <tr1/functional>
+
+using namespace test;
+using namespace paludis;
+
+namespace
+{
+ void write_to(StringListStream & s)
+ {
+ for (int n(0) ; n < 100 ; ++n)
+ {
+ s << n << std::endl;
+ usleep(100);
+ }
+
+ s.nothing_more_to_write();
+ }
+}
+
+namespace test_cases
+{
+ struct StringListStreamTest : TestCase
+ {
+ StringListStreamTest() : TestCase("string list stream") { }
+
+ void run()
+ {
+ StringListStream s;
+ s << "foo" << std::endl << "bar" << std::endl << "baz" << std::endl;
+ s.nothing_more_to_write();
+
+ std::string l;
+
+ TEST_CHECK(std::getline(s, l));
+ TEST_CHECK_EQUAL(l, "foo");
+
+ TEST_CHECK(std::getline(s, l));
+ TEST_CHECK_EQUAL(l, "bar");
+
+ TEST_CHECK(std::getline(s, l));
+ TEST_CHECK_EQUAL(l, "baz");
+
+ TEST_CHECK(! std::getline(s, l));
+ }
+ } test_string_list_stream;
+
+ struct StringListStreamThreadsTest : TestCase
+ {
+ StringListStreamThreadsTest() : TestCase("string list stream threads") { }
+
+ void run()
+ {
+ StringListStream s;
+ Thread t(std::tr1::bind(&write_to, std::tr1::ref(s)));
+
+ std::string l;
+ for (int n(0) ; n < 100 ; ++n)
+ {
+ TestMessageSuffix sx(stringify(n));
+ TEST_CHECK(std::getline(s, l));
+ TEST_CHECK_EQUAL(l, stringify(n));
+ }
+
+ TEST_CHECK(! std::getline(s, l));
+ }
+ } test_string_list_stream_thread;
+}
+
diff --git a/src/clients/cave/cmd_resolve.cc b/src/clients/cave/cmd_resolve.cc
index 885a53a..51141a3 100644
--- a/src/clients/cave/cmd_resolve.cc
+++ b/src/clients/cave/cmd_resolve.cc
@@ -33,6 +33,8 @@
#include <paludis/util/enum_iterator.hh>
#include <paludis/util/indirect_iterator-impl.hh>
#include <paludis/util/wrapped_output_iterator.hh>
+#include <paludis/util/string_list_stream.hh>
+#include <paludis/util/thread.hh>
#include <paludis/args/do_help.hh>
#include <paludis/args/escape.hh>
#include <paludis/resolver/resolver.hh>
@@ -853,6 +855,13 @@ namespace
throw InternalError(PALUDIS_HERE, "unhandled dt");
}
+ void ser_thread_func(StringListStream & ser_stream, const ResolverLists & resolution_lists)
+ {
+ Serialiser ser(ser_stream);
+ resolution_lists.serialise(ser);
+ ser_stream.nothing_more_to_write();
+ }
+
int display_resolution(
const std::tr1::shared_ptr<Environment> &,
const ResolverLists & resolution_lists,
@@ -860,9 +869,10 @@ namespace
{
Context context("When displaying chosen resolution:");
- std::stringstream ser_stream;
- Serialiser ser(ser_stream);
- resolution_lists.serialise(ser);
+ StringListStream ser_stream;
+ Thread ser_thread(std::tr1::bind(&ser_thread_func,
+ std::tr1::ref(ser_stream),
+ std::tr1::cref(resolution_lists)));
std::string command(cmdline.program_options.a_display_resolution_program.argument());
if (command.empty())
@@ -902,10 +912,15 @@ namespace
{
Context context("When performing chosen resolution:");
- std::stringstream ser_stream;
+ StringListStream ser_stream;
Serialiser ser(ser_stream);
resolution_lists.serialise(ser);
+ /* backgrounding this barfs with become_command. working out why could
+ * be a fun exercise for someone with way too much time on their hands.
+ * */
+ ser_thread_func(ser_stream, resolution_lists);
+
std::string command(cmdline.program_options.a_execute_resolution_program.argument());
if (command.empty())
command = "$CAVE execute-resolution";