/* Copyright (c) 2009-2012 250bpm s.r.o. Copyright (c) 2007-2011 iMatix Corporation Copyright (c) 2007-2011 Other contributors as noted in the AUTHORS file This file is part of Crossroads I/O project. Crossroads I/O is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. Crossroads is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this program. If not, see . */ #include "platform.hpp" #if defined XS_HAVE_WINDOWS #include "windows.hpp" #else # if HAVE_UNISTD_H # include # endif # if HAVE_SYS_TYPES_H # include # endif # if HAVE_SYS_STAT_H # include # endif # if HAVE_DLFCN_H # include # endif # if HAVE_LINK_H # include # endif # if HAVE_FCNTL_H # include # endif # if HAVE_DIRENT_H # include # endif #endif #include #include #include "ctx.hpp" #include "socket_base.hpp" #include "reaper.hpp" #include "pipe.hpp" #include "err.hpp" #include "msg.hpp" #include "prefix_filter.hpp" #include "topic_filter.hpp" xs::ctx_t::ctx_t () : tag (0xbadcafe0), starting (true), terminating (false), reaper (NULL), slot_count (0), slots (NULL), max_sockets (512), io_thread_count (1) { int rc = mailbox_init (&term_mailbox); errno_assert (rc == 0); // Plug in the standard plugins. rc = plug (prefix_filter); errno_assert (rc == 0); rc = plug (topic_filter); errno_assert (rc == 0); // Now plug in all the extensions found in plugin directory. #if defined XS_HAVE_WINDOWS // Find out the path to the plugins. char files_common [MAX_PATH]; HRESULT hr = SHGetFolderPath (NULL, CSIDL_PROGRAM_FILES_COMMON, NULL, SHGFP_TYPE_CURRENT, files_common); xs_assert (hr == S_OK); std::string path = files_common; WIN32_FIND_DATA ffd; HANDLE fh = FindFirstFile ((path + "\\xs\\plugins\\*").c_str (), &ffd); if (fh == INVALID_HANDLE_VALUE) return; while (true) { // Ignore the files without .xsp extension. std::string file = ffd.cFileName; if (file.size () < 4) goto next; if (file.substr (file.size () - 4) != ".xsp") goto next; // Load the library and locate the extension point. HMODULE dl = LoadLibrary ((path + "\\xs\\" + ffd.cFileName).c_str ()); if (!dl) goto next; file = std::string ("xsp_") + file.substr (0, file.size () - 4) + "_init"; void *(*initfn) (); *(void**)(&initfn) = GetProcAddress (dl, file.c_str ()); if (!initfn) goto next; // Plug the extension into the context. rc = plug (initfn ()); if (rc != 0) { FreeLibrary (dl); goto next; } // Store the library handle so that we can unload it // when context is terminated. opt_sync.lock (); plugins.push_back (dl); opt_sync.unlock (); next: if (FindNextFile (fh, &ffd) == 0) break; } BOOL brc = FindClose (fh); win_assert (brc != 0); #elif XS_HAVE_PLUGINS // Load all the installed plug-ins. std::string path (XS_LIBDIR_PATH); path += "/xs/plugins"; DIR *dp = opendir (path.c_str ()); if (!dp && (errno == ENOENT || errno == EACCES || errno == ENOTDIR)) return; errno_assert (dp); dirent dir, *dirp; struct stat stats; while (true) { rc = readdir_r (dp, &dir, &dirp); assert (rc == 0); if (!dirp) break; rc = lstat (dirp->d_name, &stats); if ((rc == 0) && S_ISREG (stats.st_mode)) { // Ignore the files without .xsp extension. std::string file = dir.d_name; if (file.size () < 4) continue; if (file.substr (file.size () - 4) != ".xsp") continue; // Open the specified dynamic library. std::string filename = path + "/" + file; void *dl = dlopen (filename.c_str (), RTLD_LOCAL | RTLD_NOW); if (!dl) continue; // Find the initial entry point in the library. void pointer // returned by dlsym is converted to function pointer using // "union cast". There's no other way to do that as POSIX and // C++ standards collide at this spot. file = std::string ("xsp_") + file.substr (0, file.size () - 4) + "_init"; dlerror (); union { void *pdata; void *(*pfunc) (); } initfn; initfn.pdata = dlsym (dl, file.c_str ()); if (!initfn.pdata) continue; // Plug the extension into the context. int rc = plug (initfn.pfunc ()); if (rc != 0) { dlclose (dl); continue; } // Store the library handle so that we can unload it // when context is terminated. opt_sync.lock (); plugins.push_back (dl); opt_sync.unlock (); } } closedir (dp); #endif } bool xs::ctx_t::check_tag () { return tag == 0xbadcafe0; } xs::ctx_t::~ctx_t () { // Check that there are no remaining sockets. xs_assert (sockets.empty ()); // Ask I/O threads to terminate. If stop signal wasn't sent to I/O // thread subsequent invocation of destructor would hang-up. for (io_threads_t::size_type i = 0; i != io_threads.size (); i++) io_threads [i]->stop (); // Wait till I/O threads actually terminate. for (io_threads_t::size_type i = 0; i != io_threads.size (); i++) delete io_threads [i]; // Deallocate the reaper thread object. if (reaper) delete reaper; // Deallocate the array of mailboxes. No special work is // needed as mailboxes themselves were deallocated with their // corresponding io_thread/socket objects. if (slots) free (slots); // Deallocate the termination mailbox. mailbox_close (&term_mailbox); // Remove the tag, so that the object is considered dead. tag = 0xdeadbeef; } int xs::ctx_t::terminate () { slot_sync.lock (); if (!starting) { // Check whether termination was already underway, but interrupted and now // restarted. bool restarted = terminating; terminating = true; slot_sync.unlock (); // First attempt to terminate the context. if (!restarted) { // First send stop command to sockets so that any blocking calls // can be interrupted. If there are no sockets we can ask reaper // thread to stop. slot_sync.lock (); for (sockets_t::size_type i = 0; i != sockets.size (); i++) sockets [i]->stop (); if (sockets.empty ()) reaper->stop (); slot_sync.unlock (); } // Wait till reaper thread closes all the sockets. command_t cmd; int rc = mailbox_recv (&term_mailbox, &cmd, -1); if (rc == -1 && errno == EINTR) return -1; errno_assert (rc == 0); xs_assert (cmd.type == command_t::done); slot_sync.lock (); xs_assert (sockets.empty ()); } slot_sync.unlock (); // Unload any dynamically loaded extension libraries. #if defined XS_HAVE_WINDOWS opt_sync.lock (); for (plugins_t::iterator it = plugins.begin (); it != plugins.end (); ++it) FreeLibrary (*it); opt_sync.unlock (); #elif XS_HAVE_PLUGINS opt_sync.lock (); for (plugins_t::iterator it = plugins.begin (); it != plugins.end (); ++it) dlclose (*it); opt_sync.unlock (); #endif // Deallocate the resources. delete this; return 0; } int xs::ctx_t::plug (const void *ext_) { if (!ext_) { errno = EFAULT; return -1; } // The extension is a message filter plug-in. xs_filter_t *filter = (xs_filter_t*) ext_; if (filter->type == XS_PLUGIN_FILTER && filter->version == 1) { opt_sync.lock (); filters [filter->id (NULL)] = filter; opt_sync.unlock (); return 0; } // Specified extension type is not supported by this version of // the library. errno = ENOTSUP; return -1; } int xs::ctx_t::setctxopt (int option_, const void *optval_, size_t optvallen_) { switch (option_) { case XS_MAX_SOCKETS: if (optvallen_ != sizeof (int) || *((int*) optval_) < 0) { errno = EINVAL; return -1; } opt_sync.lock (); max_sockets = *((int*) optval_); opt_sync.unlock (); break; case XS_IO_THREADS: if (optvallen_ != sizeof (int) || *((int*) optval_) < 1) { errno = EINVAL; return -1; } opt_sync.lock (); io_thread_count = *((int*) optval_); opt_sync.unlock (); break; case XS_PLUGIN: return plug (optval_); default: errno = EINVAL; return -1; } return 0; } xs::socket_base_t *xs::ctx_t::create_socket (int type_) { slot_sync.lock (); if (unlikely (starting)) { starting = false; // Initialise the array of mailboxes. Additional three slots are for // xs_term thread and reaper thread. opt_sync.lock (); int maxs = max_sockets; int ios = io_thread_count; opt_sync.unlock (); slot_count = maxs + ios + 2; slots = (mailbox_t**) malloc (sizeof (mailbox_t*) * slot_count); alloc_assert (slots); // Initialise the infrastructure for xs_term thread. slots [term_tid] = &term_mailbox; // Create the reaper thread. reaper = new (std::nothrow) reaper_t (this, reaper_tid); alloc_assert (reaper); slots [reaper_tid] = reaper->get_mailbox (); reaper->start (); // Create I/O thread objects and launch them. for (int i = 2; i != ios + 2; i++) { io_thread_t *io_thread = io_thread_t::create (this, i); errno_assert (io_thread); io_threads.push_back (io_thread); slots [i] = io_thread->get_mailbox (); io_thread->start (); } // In the unused part of the slot array, create a list of empty slots. for (int32_t i = (int32_t) slot_count - 1; i >= (int32_t) ios + 2; i--) { empty_slots.push_back (i); slots [i] = NULL; } } // Once xs_term() was called, we can't create new sockets. if (terminating) { slot_sync.unlock (); errno = ETERM; return NULL; } // If max_sockets limit was reached, return error. if (empty_slots.empty ()) { slot_sync.unlock (); errno = EMFILE; return NULL; } // Choose a slot for the socket. uint32_t slot = empty_slots.back (); empty_slots.pop_back (); // Generate new unique socket ID. int sid = ((int) max_socket_id.add (1)) + 1; // Create the socket and register its mailbox. socket_base_t *s = socket_base_t::create (type_, this, slot, sid); if (!s) { empty_slots.push_back (slot); slot_sync.unlock (); return NULL; } sockets.push_back (s); slots [slot] = s->get_mailbox (); slot_sync.unlock (); return s; } void xs::ctx_t::destroy_socket (class socket_base_t *socket_) { slot_sync.lock (); // Free the associared thread slot. uint32_t tid = socket_->get_tid (); empty_slots.push_back (tid); slots [tid] = NULL; // Remove the socket from the list of sockets. sockets.erase (socket_); // If xs_term() was already called and there are no more socket // we can ask reaper thread to terminate. if (terminating && sockets.empty ()) reaper->stop (); slot_sync.unlock (); } xs::object_t *xs::ctx_t::get_reaper () { return reaper; } xs_filter_t *xs::ctx_t::get_filter (int filter_id_) { xs_filter_t *result = NULL; opt_sync.lock (); filters_t::iterator it = filters.find (filter_id_); if (it != filters.end ()) result = it->second; opt_sync.unlock (); return result; } void xs::ctx_t::send_command (uint32_t tid_, const command_t &command_) { mailbox_send (slots [tid_], command_); } xs::io_thread_t *xs::ctx_t::choose_io_thread (uint64_t affinity_) { if (io_threads.empty ()) return NULL; // Find the I/O thread with minimum load. int min_load = -1; io_threads_t::size_type result = 0; for (io_threads_t::size_type i = 0; i != io_threads.size (); i++) { if (!affinity_ || (affinity_ & (uint64_t (1) << i))) { int load = io_threads [i]->get_load (); if (min_load == -1 || load < min_load) { min_load = load; result = i; } } } xs_assert (min_load != -1); return io_threads [result]; } int xs::ctx_t::register_endpoint (const char *addr_, endpoint_t &endpoint_) { endpoints_sync.lock (); bool inserted = endpoints.insert (endpoints_t::value_type ( std::string (addr_), endpoint_)).second; if (!inserted) { errno = EADDRINUSE; endpoints_sync.unlock (); return -1; } endpoints_sync.unlock (); return 0; } void xs::ctx_t::unregister_endpoints (socket_base_t *socket_) { endpoints_sync.lock (); endpoints_t::iterator it = endpoints.begin (); while (it != endpoints.end ()) { if (it->second.socket == socket_) { endpoints_t::iterator to_erase = it; ++it; endpoints.erase (to_erase); continue; } ++it; } endpoints_sync.unlock (); } xs::endpoint_t xs::ctx_t::find_endpoint (const char *addr_) { endpoints_sync.lock (); endpoints_t::iterator it = endpoints.find (addr_); if (it == endpoints.end ()) { endpoints_sync.unlock (); errno = ECONNREFUSED; endpoint_t empty = {NULL, options_t()}; return empty; } endpoint_t endpoint = it->second; // Increment the command sequence number of the peer so that it won't // get deallocated until "bind" command is issued by the caller. // The subsequent 'bind' has to be called with inc_seqnum parameter // set to false, so that the seqnum isn't incremented twice. endpoint.socket->inc_seqnum (); endpoints_sync.unlock (); return endpoint; } // The last used socket ID, or 0 if no socket was used so far. Note that this // is a global variable. Thus, even sockets created in different contexts have // unique IDs. xs::atomic_counter_t xs::ctx_t::max_socket_id;