Changeset 9c06054 in nscp

Timestamp:

12/07/11 07:01:41 (18 months ago)

Author:

Michael Medin <michael@…>

Branches:

master, 0.4.0, 0.4.1, 0.4.2

Children:

f085f9b

Parents:

f33c12f

Message:

• Added python unittests to make sure threading is working properly
• Simpliefied and cleaned up the command line syntax a bit
• Now --exec is the default option for client mode (but it will notify you it thought so)
• Added "command-less" execution to PythonScript so you can do --exec --script ... (without having --exec run)

Files:

• changelog (modified) (1 diff)
• include/scripts/functions.hpp (modified) (1 diff)
• modules/PythonScript/PythonScript.cpp (modified) (9 diffs)
• modules/PythonScript/PythonScript.h (modified) (1 diff)
• modules/Scheduler/Scheduler.cpp (modified) (1 diff)
• modules/Scheduler/simple_scheduler.cpp (modified) (1 diff)
• scripts/python/test_python.py (added)
• service/NSClient++.cpp (modified) (5 diffs)
• service/NSClient++.h (modified) (1 diff)
• service/cli_parser.hpp (modified) (6 diffs)
• service/settings_client.hpp (modified) (1 diff)
• service/simple_client.hpp (modified) (1 diff)

changelog

@@ -5 +5 @@
  * Fixa dependonservice LanManWorkStation (old win)
  * Fix RtlStringFromGUID problem on NT4
+
+2011-12-06 MickeM
+ * Added python unittests to make sure threading is working properly
+ * Simpliefied and cleaned up the command line syntax a bit
+ * Now --exec is the default option for client mode (but it will notify you it thought so)
+ * Added "command-less" execution to PythonScript so you can do --exec --script ... (without having --exec run)
+ * Fixed an issue related to python threading
+ * Fixed Scheduler header propgation
+ * Fixed so all Client module use "complex" API meaning headers are propagated correctly
+ * Fixed scheduler alias issue
 
 2011-11-27 MickeM

include/scripts/functions.hpp

@@ -14 +14 @@
   }
 
+  bool validate(std::wstring &error) const {
+    if (script.empty()) {
+      error = _T("No script given on command line!");
+      return false;
+    }
+    if (!boost::filesystem::exists(script)) {
+      error = _T("Script not found: ") + script.string();
+      return false;
+    }
+    if (!boost::filesystem::is_regular(script)) {
+      error = _T("Script is not a file: ") + script.string();
+      return false;
+    }
+    return true;
+  }
+
   static void push(list_type &list, std::wstring alias, boost::filesystem::wpath script) {
     list.push_back(script_container(alias, script));

modules/PythonScript/PythonScript.cpp

@@ -109 +109 @@
 python_script::python_script(unsigned int plugin_id, const std::string alias, const script_container& script) : alias(alias), plugin_id(plugin_id) {
   NSC_DEBUG_MSG_STD(_T("Loading python script: ") + script.script.string());
-  std::wcout << script.script.string() << std::endl;;
-  _exec(utf8::cvt<std::string>(script.script.string()));
+  std::wstring err;
+  if (!script.validate(err)) {
+    NSC_LOG_ERROR(err);
+    return;
+  }
+  _exec(utf8::cvt<std::string>(script.script.file_string()));
   callFunction("init", plugin_id, alias, utf8::cvt<std::string>(script.alias));
 }
@@ -162 +166 @@
       path /= _T("lib");
       PyRun_SimpleString(("sys.path.append('" + utf8::cvt<std::string>(path.string()) + "')").c_str());
+
       object ignored = exec_file(scriptfile.c_str(), localDict, localDict); 
     } catch( error_already_set e) {
       script_wrapper::log_exception();
+    } catch(...) {
+      NSC_LOG_ERROR(_T("Failed to run python script: ") + utf8::to_unicode(scriptfile));
     }
   } catch (...) {
@@ -195 +202 @@
       Py_Initialize();
       PyEval_InitThreads();
-
+      //PyEval_ReleaseLock();
+
+      PyThreadState *state;
+      state = PyEval_SaveThread();
         //PyThreadState *mainThreadState = PyThreadState_Get();
         //PyThreadState_Clear
-        //PyGILState_Release(mainThreadState);
+        //PyThreadState_(mainThreadState);
 
       do_init = true;
@@ -220 +230 @@
 
       }
+      //PyEval_ReleaseLock();
       BOOST_FOREACH(script_container &script, scripts_) {
         instances_.push_back(boost::shared_ptr<python_script>(new python_script(get_id(), utf8::cvt<std::string>(alias), script)));
       }
-      PyEval_ReleaseLock();
 
     } catch (std::exception &e) {
@@ -255 +265 @@
 }
 
-NSCAPI::nagiosReturn PythonScript::execute_and_load_python(std::list<std::wstring> args) {
-  try {
-    po::options_description desc;
+NSCAPI::nagiosReturn PythonScript::execute_and_load_python(std::list<std::wstring> args, std::wstring &message) {
+  try {
+    po::options_description desc("Options for the following commands: (exec, execute)");
     boost::program_options::variables_map vm;
     std::wstring file;
     desc.add_options()
+      ("help", "Display help")
       ("script", po::wvalue<std::wstring>(&file), "The script to run")
       ("file", po::wvalue<std::wstring>(&file), "The script to run")
@@ -269 +280 @@
     po::store(parsed, vm);
     po::notify(vm);
+
+    if (vm.count("help") > 0) {
+      std::stringstream ss;
+      ss << desc;
+      message = utf8::to_unicode(ss.str());
+      return NSCAPI::returnUNKNOWN;
+    }
 
     boost::optional<boost::filesystem::wpath> ofile = find_file(file);
@@ -276 +294 @@
     python_script script(get_id(), "", sc);
     script.callFunction("__main__");
+    message = _T("Script execute successfully...");
+    return NSCAPI::returnOK;
   } catch (const std::exception &e) {
-    NSC_LOG_ERROR_STD(_T("Failed to execute script ") + utf8::cvt<std::wstring>(e.what()));
-  } catch (...) {
-    NSC_LOG_ERROR_STD(_T("Failed to execute script..."));
+    message = _T("Failed to execute script ") + utf8::to_unicode(e.what());
+    NSC_LOG_ERROR_STD(message);
+    return NSCAPI::returnUNKNOWN;
+  } catch (...) {
+    message = _T("Failed to execute script ");
+    NSC_LOG_ERROR_STD(message);
+    return NSCAPI::returnUNKNOWN;
   }
 }
@@ -344 +368 @@
 NSCAPI::nagiosReturn PythonScript::commandRAWLineExec(const wchar_t* char_command, const std::string &request, std::string &response) {
   std::wstring command = char_command;
-  if (command == _T("execute-and-load-python") || command == _T("execute-python") || command == _T("run")) {
+  if (command == _T("help")) {
+    std::list<std::wstring> args;
+    args.push_back(_T("--help"));
+    std::wstring result;
+    int ret = execute_and_load_python(args, result);
+    nscapi::functions::create_simple_exec_response<std::wstring>(_T("help"), ret, result, response);
+    return ret;
+  } else if (command == _T("execute-and-load-python") || command == _T("execute-python") || command == _T("run") || command == _T("execute") || command == _T("exec")|| command == _T("")) {
     nscapi::functions::decoded_simple_command_data data = nscapi::functions::parse_simple_exec_request(char_command, request);
-    return execute_and_load_python(data.args);
+    if (data.command == _T("") && data.args.size() == 0)
+      return NSCAPI::returnIgnored;
+    std::wstring result;
+    int ret = execute_and_load_python(data.args, result);
+    nscapi::functions::create_simple_exec_response(data.command, ret, result, response);
+    return ret;
   }
   boost::shared_ptr<script_wrapper::function_wrapper> inst = script_wrapper::function_wrapper::create(get_id());
@@ -398 +434 @@
     std::wstring src, cmd, msg, perf;
     int code = nscapi::functions::parse_simple_submit_request(request, src, cmd, msg, perf);
-    NSC_LOG_ERROR_STD(_T(" --- command: ") + cmd);
     int ret = inst->handle_simple_message(chnl, to_string(src), to_string(cmd), code, msg, perf);
     nscapi::functions::create_simple_submit_response(channel, cmd, ret, _T(""), response);

modules/PythonScript/PythonScript.h

@@ -88 +88 @@
   NSCAPI::nagiosReturn handleRAWNotification(const std::wstring &channel, std::string &request, std::string &response);
 
-  NSCAPI::nagiosReturn execute_and_load_python(std::list<std::wstring> args);
-  //NSCAPI::nagiosReturn RunLUA(const unsigned int argLen, wchar_t **char_args, std::wstring &message, std::wstring &perf);
-  //NSCAPI::nagiosReturn extract_return(Lua_State &L, int arg_count,  std::wstring &message, std::wstring &perf);
-
-  //script_wrapper::lua_handler
-  //void register_command(script_wrapper::lua_script* script, std::wstring command, std::wstring function);
+  NSCAPI::nagiosReturn execute_and_load_python(std::list<std::wstring> args, std::wstring &message);
 
 private:

modules/Scheduler/Scheduler.cpp

@@ -41 +41 @@
     schedule_map schedules;
     sh::settings_registry settings(get_settings_proxy());
-    settings.set_alias(_T("scheduler"), alias);
+    settings.set_alias(alias, _T("scheduler"));
 
     settings.alias().add_path_to_settings()

modules/Scheduler/simple_scheduler.cpp

@@ -78 +78 @@
 
   void simple_scheduler::thread_proc(int id) {
-    int iteration = 0;
-    schedule_queue_type::value_type instance;
-    while (!stop_requested_) {
-      instance = queue_.pop();
-      if (!instance) {
-        boost::unique_lock<boost::mutex> lock(idle_thread_mutex_);
-        idle_thread_cond_.wait(lock);
-        continue;
+    try {
+      int iteration = 0;
+      schedule_queue_type::value_type instance;
+      while (!stop_requested_) {
+        instance = queue_.pop();
+        if (!instance) {
+          boost::unique_lock<boost::mutex> lock(idle_thread_mutex_);
+          idle_thread_cond_.wait(lock);
+          continue;
+        }
+
+        try {
+          boost::posix_time::time_duration off = now() - (*instance).time;
+          if (off.total_seconds() > error_threshold_) {
+            log_error(_T("Ran scheduled item ") + to_wstring((*instance).schedule_id) + _T(" ") + to_wstring(off.total_seconds()) + _T(" seconds to late from thread ") + to_wstring(id));
+          }
+          boost::thread::sleep((*instance).time);
+        } catch (boost::thread_interrupted  &e) {
+          if (!queue_.push(*instance))
+            log_error(_T("ERROR"));
+          if (stop_requested_) {
+            log_error(_T("Terminating thread: ") + to_wstring(id));
+            return;
+          }
+          continue;
+        } catch (...) {
+          if (!queue_.push(*instance))
+            log_error(_T("ERROR"));
+          continue;
+        }
+
+        boost::posix_time::ptime now_time = now();
+        boost::optional<target> item = get_task((*instance).schedule_id);
+        if (item) {
+          try {
+            if (handler_)
+              handler_->handle_schedule(*item);
+            reschedule(*item,now_time);
+          } catch (...) {
+            log_error(_T("UNKNOWN ERROR RUNING TASK: "));
+            reschedule(*item);
+          }
+        } else {
+          log_error(_T("Task not found: ") + to_wstring((*instance).schedule_id));
+        }
       }
+    } catch (const std::exception &e) {
+      log_error(_T("Exception in scheduler thread (thread will be killed): ") + utf8::to_unicode(e.what()));
+    } catch (...) {
+      log_error(_T("Exception in scheduler thread (thread will be killed)"));
+    }
 
-      try {
-        boost::posix_time::time_duration off = now() - (*instance).time;
-        if (off.total_seconds() > error_threshold_) {
-          log_error(_T("Ran scheduled item ") + to_wstring((*instance).schedule_id) + _T(" ") + to_wstring(off.total_seconds()) + _T(" seconds to late from thread ") + to_wstring(id));
-        }
-        boost::thread::sleep((*instance).time);
-      } catch (boost::thread_interrupted  &e) {
-        if (!queue_.push(*instance))
-          log_error(_T("ERROR"));
-        if (stop_requested_) {
-          log_error(_T("Terminating thread: ") + to_wstring(id));
-          return;
-        }
-        continue;
-      } catch (...) {
-        if (!queue_.push(*instance))
-          log_error(_T("ERROR"));
-        continue;
-      }
-
-      boost::posix_time::ptime now_time = now();
-      boost::optional<target> item = get_task((*instance).schedule_id);
-      if (item) {
-        try {
-          if (handler_)
-            handler_->handle_schedule(*item);
-          reschedule(*item,now_time);
-        } catch (...) {
-          log_error(_T("UNKNOWN ERROR RUNING TASK: "));
-          reschedule(*item);
-        }
-      } else {
-        log_error(_T("Task not found: ") + to_wstring((*instance).schedule_id));
-      }
-    }
   }
 

service/NSClient++.cpp

@@ -689 +689 @@
   return true;
 }
-bool NSClientT::boot_load_plugins(bool boot) {
+bool NSClientT::boot_load_all_plugins() {
   LOG_DEBUG_CORE(_T("booting::loading plugins"));
   try {
@@ -717 +717 @@
   } catch (settings::settings_exception e) {
     LOG_ERROR_CORE_STD(_T("Settings exception when loading modules: ") + e.getMessage());
+    return false;
   } catch (...) {
     LOG_ERROR_CORE_STD(_T("Unknown exception when loading plugins"));
     return false;
   }
+  return true;
+}
+
+bool NSClientT::boot_load_plugin(std::wstring plugin) {
+  try {
+    if (plugin.length() > 4 && plugin.substr(plugin.length()-4) == _T(".dll"))
+      plugin = plugin.substr(0, plugin.length()-4);
+
+    std::wstring plugin_file = NSCPlugin::get_plugin_file(plugin);
+    boost::filesystem::wpath pluginPath = expand_path(_T("${module-path}"));
+    boost::filesystem::wpath file = pluginPath / plugin_file;
+    if (boost::filesystem::is_regular(file)) {
+      plugin_type plugin = addPlugin(file, _T(""));
+    } else {
+      LOG_ERROR_CORE_STD(_T("Failed to load: ") + std::wstring(plugin));
+      return false;
+    }
+  } catch (const NSPluginException &e) {
+    LOG_ERROR_CORE_STD(_T("Module (") + e.file_ + _T(") was not found: ") + e.error_);
+    return false;
+  } catch(const std::exception &e) {
+    LOG_ERROR_CORE_STD(_T("Module (") + plugin + _T(") was not found: ") + utf8::to_unicode(e.what()));
+    return false;
+  } catch(...) {
+    LOG_ERROR_CORE_STD(_T("Module (") + plugin + _T(") was not found..."));
+    return false;
+  }
+  return true;
+}
+bool NSClientT::boot_start_plugins(bool boot) {
   try {
     loadPlugins(boot?NSCAPI::normalStart:NSCAPI::dontStart);
@@ -848 +879 @@
  */
 void NSClientT::unloadPlugins(bool unloadLoggers) {
+  if (unloadLoggers)
   {
     boost::unique_lock<boost::shared_mutex> writeLock(m_mutexRW, boost::get_system_time() + boost::posix_time::seconds(10));
@@ -854 +886 @@
       return;
     }
-    if (unloadLoggers)
-      logger_master_.remove_all_plugins();
+    logger_master_.remove_all_plugins();
   }
   {
@@ -1467 +1498 @@
   service_name_ = service_name;
   boot_init();
-  boot_load_plugins(true);
+  boot_load_all_plugins();
+  boot_start_plugins(true);
 /*
   DWORD dwSessionId = remote_processes::getActiveSessionId();

service/NSClient++.h

@@ -132 +132 @@
   // Service helper functions
   bool boot_init();
-  bool boot_load_plugins(bool boot);
+  bool boot_load_all_plugins();
+  bool boot_load_plugin(std::wstring plugin);
+  bool boot_start_plugins(bool boot);
   bool exitCore(bool boot);
   void set_settings_context(std::wstring context) { context_ = context; }

service/cli_parser.hpp

@@ -75 +75 @@
 
     client.add_options()
-      ("exec,e", po::value<std::wstring>(), "Run a command (execute)")
+      ("exec,e", po::value<std::wstring>()->implicit_value(_T("")), "Run a command (execute)")
+      ("boot,b", "Boot the client before executing command (similar as running the command from test)")
       ("query,q", po::value<std::wstring>(), "Run a query with a given name")
       ("submit,s", po::value<std::wstring>(), "Name of query to ask")
@@ -335 +336 @@
 
       std::wstring command;
-      enum modes { exec, query, submit};
-      modes mode;
+      enum modes { exec, query, submit, none};
+      modes mode = none;
 
       if (vm.count("exec")) {
@@ -346 +347 @@
         mode = query;
       }
+      if (vm.count("submit")) {
+        command = vm["submit"].as<std::wstring>();
+        mode = submit;
+      }
 
       if (vm.count("module"))
         module = vm["module"].as<std::wstring>();
+
+      bool boot = false;
+      if (vm.count("boot"))
+        boot = true;
 
       std::vector<std::wstring> kvp_args;
@@ -382 +391 @@
         mainClient.log_info(__FILE__, __LINE__, _T("Command: ") + command);
         mainClient.log_info(__FILE__, __LINE__, _T("Mode: ") + strEx::itos(mode));
+        mainClient.log_info(__FILE__, __LINE__, _T("Boot: ") + strEx::itos(boot));
         std::wstring args;
         BOOST_FOREACH(std::wstring s, arguments)
@@ -388 +398 @@
       }
       core_->boot_init();
-      if (module.empty()) {
-        core_->boot_load_plugins(false);
-      }
+      if (module.empty())
+        core_->boot_load_all_plugins();
+      else
+        core_->boot_load_plugin(module);
+      core_->boot_start_plugins(boot);
       int ret = 0;
       std::list<std::wstring> resp;
+      if (mode == none) {
+        mode = exec;
+        std::wcerr << _T("Since no mode was specified assuming --exec (other options are --query and --submit)") << std::endl;
+      }
       if (mode == query) {
         ret = mainClient.simple_query(module, command, arguments, resp);
+      } else if (mode == exec) {
+        ret = mainClient.simple_exec(module, command, arguments, resp);
+        if (ret == NSCAPI::returnIgnored) {
+          ret = 1;
+          std::wcout << _T("Command not found (by module): ") << command << std::endl;
+          mainClient.simple_exec(module, _T("help"), arguments, resp);
+        }
       } else {
-        if (command.empty()) {
-          mainClient.simple_exec(module, _T("help"), arguments, resp);
-          ret = 1;
-        } else {
-          ret = mainClient.simple_exec(module, command, arguments, resp);
-          if (ret == NSCAPI::returnIgnored) {
-            ret = 1;
-            std::wcout << _T("Command not found (by module): ") << command << std::endl;
-            mainClient.simple_exec(module, _T("help"), arguments, resp);
-          }
-        }
-      }
-      mainClient.exitCore(false);
+        std::wcerr << _T("Need to specify one of --exec, --query or --submit") << std::endl;
+      }
+      mainClient.exitCore(boot);
 
       BOOST_FOREACH(std::wstring r, resp) {
@@ -415 +428 @@
       return ret;
     } catch(std::exception & e) {
-      mainClient.log_error(__FILE__, __LINE__, std::string("Client: Unable to parse command line: ") + e.what());
+      std::wcerr << _T("Client: Unable to parse command line: ") << utf8::to_unicode(e.what()) << std::endl;
       return 1;
     } catch(...) {
-      mainClient.log_error(__FILE__, __LINE__, "Unknown exception parsing command line");
+      std::wcerr << _T("Client: Unable to parse command line: UNKNOWN") << std::endl;
       return 1;
     }

service/settings_client.hpp

@@ -26 +26 @@
         return;
       }
-      if (!core_->boot_load_plugins(false)) {
-        std::wcout << _T("boot::load_plugins failed!") << std::endl;
+      if (!core_->boot_load_all_plugins()) {
+        std::wcout << _T("boot::load_all_plugins failed!") << std::endl;
+        return;
+      }
+      if (!core_->boot_start_plugins(false)) {
+        std::wcout << _T("boot::start_plugins failed!") << std::endl;
         return;
       }

service/simple_client.hpp

@@ -19 +19 @@
         return;
       }
-      if (!core_->boot_load_plugins(true)) {
-        core_->log_error(__FILE__, __LINE__, _T("Service failed to load plugin"));
+      if (!core_->boot_load_all_plugins()) {
+        core_->log_error(__FILE__, __LINE__, _T("Service failed to load plugins"));
+        return;
+      }
+      if (!core_->boot_start_plugins(true)) {
+        core_->log_error(__FILE__, __LINE__, _T("Service failed to start plugins"));
         return;
       }