Belle II Software  release-05-01-25
ERecoMasterCallback.cc
1 #include "daq/expreco/ERecoMasterCallback.h"
2 
3 #include "daq/expreco/ERecoRunControlCallback.h"
4 
5 #include "daq/rfarm/manager/RFCommand.h"
6 #include "daq/rfarm/manager/RfNodeInfo.h"
7 #include "daq/rfarm/manager/RfUnitInfo.h"
8 
9 #include <daq/slc/nsm/NSMCommunicator.h>
10 #include <daq/slc/runcontrol/RCCommand.h>
11 #include <daq/slc/runcontrol/RCHandlerException.h>
12 #include <daq/slc/system/Time.h>
13 
14 #include <daq/slc/base/StringUtil.h>
15 #include <daq/slc/base/TimeoutException.h>
16 
17 using namespace Belle2;
18 
19 ERecoMasterCallback::ERecoMasterCallback(ConfigFile& config) : m_callback(NULL)
20 {
21  reg(RFCommand::CONFIGURE);
22  reg(RFCommand::UNCONFIGURE);
23  reg(RFCommand::START);
24  reg(RFCommand::STOP);
25  reg(RFCommand::RESUME);
26  reg(RFCommand::PAUSE);
27  reg(RFCommand::RESTART);
28  reg(RFCommand::STATUS);
29 
30  m_rcnode = NSMNode(config.get("ereco.rcnode"));
31  LogFile::debug("ERecoMaster started: Bridge Node = %s", m_rcnode.getName().c_str());
32 }
33 
34 void ERecoMasterCallback::initialize(const DBObject& obj)
35 {
36  configure(obj);
37 }
38 
39 void ERecoMasterCallback::configure(const DBObject& obj)
40 {
41  m_nodes = NSMNodeList();
42  m_dataname = StringList();
43  const std::string format = obj("system").getText("nsmdata");
44  // 0. Register DqmServer
45  if (obj.hasObject("dqmserver")) {
46  m_nodes.push_back(NSMNode(obj("dqmserver").getText("nodename")));
47  }
48  // 1. Register distributor
49  std::string nodename = obj("distributor").getText("nodename");
50  m_nodes.push_back(NSMNode(nodename));
51  addData(nodename, format);
52  // 2. Register event processors
53  const DBObject& processor(obj("processor"));
54  int maxnodes = processor.getInt("nnodes");
55  int idbase = processor.getInt("idbase");
56  std::string hostbase = processor.getText("nodebase");
57  std::string badlist = processor.getText("badlist");
58  char idname[3];
59  for (int i = 0; i < maxnodes; i++) {
60  sprintf(idname, "%2.2d", idbase + i);
61  if (badlist.size() == 0 || StringUtil::find(badlist, idname)) {
62  nodename = StringUtil::form("evp_%s%2.2d", hostbase.c_str(), idbase + i);
63  m_nodes.push_back(NSMNode(nodename));
64  addData(nodename, format);
65  }
66  }
67  // 3. Register Event Sampler
68  nodename = obj("eventsampler").getText("nodename");
69  m_nodes.push_back(NSMNode(nodename));
70 
71  add(new NSMVHandlerInt("nnodes", true, false, m_nodes.size()));
72  int i = 0;
73  std::string rcconf = obj.getName();
74  if (StringUtil::find(rcconf, "@")) {
75  rcconf = StringUtil::split(rcconf, '@')[1];
76  }
77  for (NSMNodeList::iterator it = m_nodes.begin();
78  it != m_nodes.end(); it++) {
79  NSMNode& node(*it);
80  RCState s(node.getState());
81  std::string vname = StringUtil::form("node[%d]", i++);
82  add(new NSMVHandlerText(vname + ".rcstate", true, false, s.getLabel()));
83  add(new NSMVHandlerText(vname + ".rcconfig", true, false, rcconf));
84  add(new NSMVHandlerText(vname + ".name", true, false,
85  StringUtil::tolower(node.getName())));
86  vname = StringUtil::tolower(node.getName());
87  add(new NSMVHandlerText(vname + ".rcstate", true, false, s.getLabel()));
88  add(new NSMVHandlerText(vname + ".rcconfig", true, false, rcconf));
89  }
90 }
91 
92 void ERecoMasterCallback::setState(NSMNode& node, const RCState& state)
93 {
94  node.setState(state);
95  int i = 0;
96  for (NSMNodeList::iterator it = m_nodes.begin();
97  it != m_nodes.end(); it++) {
98  NSMNode& rnode(*it);
99  if (StringUtil::toupper(rnode.getName()) == StringUtil::toupper(node.getName())) {
100  std::string vname = StringUtil::form("node[%d].rcstate", i);
101  set(vname, state.getLabel());
102  break;
103  }
104  i++;
105  }
106  std::string vname = StringUtil::tolower(node.getName()) + ".rcstate";
107  set(vname, state.getLabel());
108 }
109 
110 void ERecoMasterCallback::monitor()
111 {
112  if (!m_callback->getData().isAvailable()) return;
113  RfUnitInfo* unitinfo = (RfUnitInfo*)m_callback->getData().get();
114  unitinfo->nnodes = m_dataname.size();
115  unitinfo->updatetime = Time().get();
116  unitinfo->rcstate = getNode().getState().getId();
117  int i = 0;
118  for (StringList::iterator it = m_dataname.begin();
119  it != m_dataname.end(); it++) {
120  NSMData& data(getData(*it));
121  try {
122  if (data.isAvailable()) {
123  RfUnitInfo::RfNodeInfo* nodeinfo_o = &unitinfo->nodeinfo[i];
124  RfNodeInfo* nodeinfo_i = (RfNodeInfo*)data.get();
125  memcpy(nodeinfo_o, nodeinfo_i, sizeof(RfNodeInfo));
126  }
127  } catch (const NSMHandlerException& e) {}
128  i++;
129  }
130  static unsigned long long count = 0;
131  for (NSMNodeList::iterator it = m_nodes.begin(); it != m_nodes.end(); it++) {
132  NSMNode& node(*it);
133  if (count % 60 == 59 || node.getState() == RCState::UNKNOWN) {
134  if (NSMCommunicator::send(NSMMessage(node, RFCommand::STATUS))) {
135  } else {
136  setState(node, RCState::UNKNOWN);
137  }
138  }
139  }
140  count++;
141 }
142 
143 void ERecoMasterCallback::ok(const char* nodename, const char* data)
144 {
145  for (NSMNodeList::iterator it = m_nodes.begin();
146  it != m_nodes.end(); it++) {
147  NSMNode& node(*it);
148  if (node.getName() == nodename) {
149  if (strcmp(data, "Configured") == 0) {
150  setState(node, RCState::READY_S);
151  } else if (strcmp(data, "Unconfigured") == 0) {
152  setState(node, RCState::NOTREADY_S);
153  } else if (strcmp(data, "Stopped") == 0) {
154  setState(node, RCState::READY_S);
155  } else if (strcmp(data, "Running") == 0) {
156  setState(node, RCState::RUNNING_S);
157  } else if (strcmp(data, "Idle") == 0) {
158  setState(node, RCState::PAUSED_S);
159  }
160  LogFile::debug("OK << %s (%s)", nodename, data);
161  return;
162  }
163  }
164  LogFile::warning("OK from unknown node %s (%s)", nodename, data);
165 }
166 
167 void ERecoMasterCallback::error(const char* nodename, const char* data)
168 {
169  for (NSMNodeList::iterator it = m_nodes.begin();
170  it != m_nodes.end(); it++) {
171  NSMNode& node(*it);
172  if (node.getName() == nodename) {
173  LogFile::error("ERROR << %s (%s)", nodename, data);
174  setState(node, RCState::NOTREADY_S);
175  return;
176  }
177  }
178  LogFile::warning("Error from unknwon node %s : %s)", nodename, data);
179 }
180 
181 void ERecoMasterCallback::load(const DBObject& db, const std::string& runtype)
182 {
183  for (NSMNodeList::iterator it = m_nodes.begin(); it != m_nodes.end(); it++) {
184  NSMNode& node(*it);
185  printf("Loading : %s\n", node.getName().c_str());
186  if (node.getName().find("EVP") == std::string::npos) {
187  while (true) {
188  bool configured = true;
189  for (NSMNodeList::iterator it2 = m_nodes.begin();
190  it2 != it; it2++) {
191  if (it2->getState() != RCState::READY_S)
192  configured = false;
193  }
194  if (configured) break;
195  try {
196  perform(NSMCommunicator::select(30));
197  } catch (const TimeoutException& e) {}
198  }
199  }
200  if (node.getState() != RCState::READY_S) {
201  printf("ERecoMasterCallback::load : loading %s\n", (node.getName()).c_str());
202  if (NSMCommunicator::send(NSMMessage(node, RCCommand::LOAD))) {
203  setState(node, RCState::LOADING_TS);
204  LogFile::debug("%s >> LOADING", node.getName().c_str());
205  } else {
206  throw (RCHandlerException("Failed to configure %s", node.getName().c_str()));
207  }
208  } else {
209  LogFile::debug("%s is READY", node.getName().c_str());
210  }
211  }
212  while (true) {
213  bool configured = true;
214  for (NSMNodeList::iterator it = m_nodes.begin(); it != m_nodes.end(); it++) {
215  if (it->getState() != RCState::READY_S) configured = false;
216  }
217  if (configured) break;
218  try {
219  perform(NSMCommunicator::select(30));
220  } catch (const TimeoutException& e) {}
221  }
222  if (m_callback != NULL)
223  m_callback->setState(RCState::READY_S);
224  RCCallback::setState(RCState::READY_S);
225 }
226 
227 void ERecoMasterCallback::abort()
228 {
229  for (NSMNodeList::reverse_iterator it = m_nodes.rbegin();
230  it != m_nodes.rend(); it++) {
231  NSMNode& node(*it);
232  if (node.getName().find("EVP") == std::string::npos) {
233  while (true) {
234  bool unconfigured = true;
235  for (NSMNodeList::reverse_iterator it2 = m_nodes.rbegin();
236  it2 != it; it2++) {
237  if (it2->getState() != RCState::NOTREADY_S)
238  unconfigured = false;
239  }
240  if (unconfigured) break;
241  try {
242  perform(NSMCommunicator::select(30));
243  } catch (const TimeoutException& e) {}
244  }
245  }
246  if (node.getState() != RCState::NOTREADY_S) {
247  if (NSMCommunicator::send(NSMMessage(node, RCCommand::ABORT))) {
248  setState(node, RCState::ABORTING_RS);
249  LogFile::debug("%s >> ABORTING", node.getName().c_str());
250  } else {
251  throw (RCHandlerException("Failed to unconfigure %s", node.getName().c_str()));
252  }
253  } else {
254  LogFile::debug("%s is NOTREADY", node.getName().c_str());
255  }
256  }
257  while (true) {
258  bool unconfigured = true;
259  for (NSMNodeList::reverse_iterator it = m_nodes.rbegin(); it != m_nodes.rend(); it++) {
260  if (it->getState() != RCState::NOTREADY_S) unconfigured = false;
261  }
262  if (unconfigured) break;
263  try {
264  perform(NSMCommunicator::select(30));
265  } catch (const TimeoutException& e) {}
266  }
267  if (m_callback != NULL)
268  m_callback->setState(RCState::NOTREADY_S);
269  RCCallback::setState(RCState::NOTREADY_S);
270 }
271 
272 void ERecoMasterCallback::start(int expno, int runno)
273 {
274  int pars[] = {expno, runno};
275  std::string script;
276  if (getNode().getName().find("_") != std::string::npos) {
277  get("hlt.script", script);
278  // script = "run_processor.py";
279  } else {
280  LogFile::debug("asking database : node = %s", m_rcnode.getName().c_str());
281  get(m_rcnode, "RUNCONTROL@hlt.script", script);
282  // get(NSMNode("BERECO1"), "RUNCONTROL@hlt.script", script);
283  // get(m_rcnode, "hlt.script", script, 30);
284  script = "run_" + script + ".py";
285  printf("ERECO script = %s\n", script.c_str());
286  }
287  for (NSMNodeList::iterator it = m_nodes.begin(); it != m_nodes.end(); it++) {
288  NSMNode& node(*it);
289  if (node.getName().find("EVP") == std::string::npos) {
290  while (true) {
291  bool configured = true;
292  for (NSMNodeList::iterator it2 = m_nodes.begin();
293  it2 != it; it2++) {
294  if (it2->getState() != RCState::RUNNING_S)
295  configured = false;
296  }
297  if (configured) break;
298  try {
299  perform(NSMCommunicator::select(30));
300  } catch (const TimeoutException& e) {}
301  }
302  }
303  if (node.getState() != RCState::RUNNING_S) {
304  if (NSMCommunicator::send(NSMMessage(node, RCCommand::START, 2, pars, script))) {
305  setState(node, RCState::STARTING_TS);
306  LogFile::debug("%s >> STARTING (script=%s)", node.getName().c_str(), script.c_str());
307  } else {
308  throw (RCHandlerException("Failed to configure %s", node.getName().c_str()));
309  }
310  } else {
311  LogFile::debug("%s is RUNNING", node.getName().c_str());
312  }
313  }
314  while (true) {
315  bool configured = true;
316  for (NSMNodeList::iterator it = m_nodes.begin(); it != m_nodes.end(); it++) {
317  if (it->getState() != RCState::RUNNING_S) configured = false;
318  }
319  if (configured) break;
320  try {
321  perform(NSMCommunicator::select(30));
322  } catch (const TimeoutException& e) {}
323  }
324  if (m_callback != NULL)
325  m_callback->setState(RCState::RUNNING_S);
326  RCCallback::setState(RCState::RUNNING_S);
327 }
328 
329 void ERecoMasterCallback::stop()
330 {
331  for (NSMNodeList::reverse_iterator it = m_nodes.rbegin();
332  it != m_nodes.rend(); it++) {
333  NSMNode& node(*it);
334  if (node.getName().find("EVP") == std::string::npos) {
335  while (true) {
336  bool unconfigured = true;
337  for (NSMNodeList::reverse_iterator it2 = m_nodes.rbegin();
338  it2 != it; it2++) {
339  if (it2->getState() != RCState::READY_S)
340  unconfigured = false;
341  }
342  if (unconfigured) break;
343  try {
344  perform(NSMCommunicator::select(30));
345  } catch (const TimeoutException& e) {}
346  }
347  }
348  if (node.getState() != RCState::READY_S) {
349  if (NSMCommunicator::send(NSMMessage(node, RCCommand::STOP))) {
350  setState(node, RCState::STOPPING_TS);
351  LogFile::debug("%s >> STOPPING", node.getName().c_str());
352  } else {
353  throw (RCHandlerException("Failed to stop %s", node.getName().c_str()));
354  }
355  } else {
356  LogFile::debug("%s is READY", node.getName().c_str());
357  }
358  }
359  while (true) {
360  bool unconfigured = true;
361  for (NSMNodeList::reverse_iterator it = m_nodes.rbegin(); it != m_nodes.rend(); it++) {
362  if (it->getState() != RCState::READY_S) unconfigured = false;
363  }
364  if (unconfigured) break;
365  try {
366  perform(NSMCommunicator::select(30));
367  } catch (const TimeoutException& e) {}
368  }
369  if (m_callback != NULL)
370  m_callback->setState(RCState::READY_S);
371  RCCallback::setState(RCState::READY_S);
372 }
373 
374 bool ERecoMasterCallback::pause()
375 {
376  return true;
377 }
378 
379 bool ERecoMasterCallback::resume(int /*subno*/)
380 {
381  return true;
382 }
383 
384 void ERecoMasterCallback::recover(const DBObject& obj, const std::string& runtype)
385 {
386  abort();
387  load(obj, runtype);
388 }
389 
390 void ERecoMasterCallback::addData(const std::string& dataname, const std::string& format)
391 {
392  openData(dataname, format);
393  int i = (int)m_dataname.size();
394  std::string vname = StringUtil::form("data[%d].name", i);
395  add(new NSMVHandlerText(vname, true, false, StringUtil::tolower(dataname)));
396  vname = StringUtil::form("data[%d].format", i);
397  add(new NSMVHandlerText(vname, true, false, format));
398  m_dataname.push_back(dataname);
399 }
400 
401 bool ERecoMasterCallback::perform(NSMCommunicator& com)
402 {
403  const NSMMessage msg = com.getMessage();
404  if (RCCallback::perform(com)) {
405  return true;
406  }
407  const RFCommand cmd = msg.getRequestName();
408  try {
409  if (cmd == RFCommand::CONFIGURE) {
410  load(getDBObject(), "");
411  return true;
412  } else if (cmd == RFCommand::UNCONFIGURE) {
413  abort();
414  return true;
415  } else if (cmd == RFCommand::START) {
416  return true;
417  } else if (cmd == RFCommand::STOP) {
418  return true;
419  } else if (cmd == RFCommand::RESTART) {
420  return true;
421  } else if (cmd == RFCommand::PAUSE) {
422  return true;
423  } else if (cmd == RFCommand::RESUME) {
424  return true;
425  } else if (cmd == RFCommand::STATUS) {
426  return true;
427  }
428  } catch (const RCHandlerException& e) {
429  std::string emsg = StringUtil::form("%s. aborting", e.what());
430  LogFile::error(emsg);
431  reply(NSMMessage(NSMCommand::ERROR, emsg));
432  try {
433  RCCallback::setState(RCState::ABORTING_RS);
434  abort();
435  RCCallback::setState(RCState::NOTREADY_S);
436  } catch (const RCHandlerException& e) {
437  std::string emsg = StringUtil::form("Failed to abort : %s", e.what());
438  LogFile::fatal(emsg);
439  }
440  } catch (const std::exception& e) {
441  LogFile::fatal("Unknown exception: %s. terminating process", e.what());
442  }
443  return false;
444 }
Belle2::ObjectInfo::getName
TString getName(const TObject *obj)
human-readable name (e.g.
Definition: ObjectInfo.cc:38
Belle2::RfUnitInfo
Definition: RfUnitInfo.h:18
prepareAsicCrosstalkSimDB.e
e
aux.
Definition: prepareAsicCrosstalkSimDB.py:53
Belle2::RCHandlerException
Definition: RCHandlerException.h:12
Belle2::NSMNode
Definition: NSMNode.h:14
Belle2::NSMVHandlerInt
Definition: NSMVHandler.h:81
Belle2::RFCommand
Definition: RFCommand.h:14
Belle2::DBObject
Definition: DBObject.h:14
Belle2::NSMData
Definition: NSMData.h:24
Belle2::NSMCommunicator
Definition: NSMCommunicator.h:25
Belle2::NSMMessage
Definition: NSMMessage.h:29
Belle2
Abstract base class for different kinds of events.
Definition: MillepedeAlgorithm.h:19
Belle2::RfUnitInfo::RfNodeInfo
Definition: RfUnitInfo.h:19
Belle2::NSMHandlerException
Definition: NSMHandlerException.h:12
Belle2::Time
Definition: Time.h:14
Belle2::TimeoutException
Definition: TimeoutException.h:12
Belle2::ConfigFile
Definition: ConfigFile.h:15
Belle2::NSMVHandlerText
Definition: NSMVHandler.h:151
Belle2::RfNodeInfo
Definition: RfNodeInfo.h:17
Belle2::RCState
Definition: RCState.h:12