Bot debugging

docs/intelmqctl.md

@@ -8,6 +8,7 @@
  * [status](#status)
  * [restart](#restart)
  * [reload](#reload)
+ * [run](#run)
  * [disable](#disable)
  * [enable](#enable)
  * [Manage the botnet](#manage-the-botnet)
@@ -118,6 +119,113 @@ If the bot is not running, we can't reload it:
 intelmqctl: file-output was NOT RUNNING.
 ```
 
+### run
+
+Run a bot directly for debugging purpose. Command temporarily leverages the logging level to DEBUG so that all the ```self.logger.debug("message")``` in the bot will get displayed.
+
+If launched with no arguments, the bot will call its init method and start processing messages as usual – but you see everything happens.
+
+```bash
+> intelmqctl run file-output
+file-output: RestAPIOutputBot initialized with id file-output and version 3.5.2 as process 12345.
+file-output: Bot is starting.
+file-output: Loading source pipeline and queue 'file-output-queue'.
+file-output: Connected to source queue.
+file-output: No destination queues to load.
+file-output: Pipeline ready.
+file-output: Waiting for incoming message.
+```
+
+Should you get lost any time, just use the **--help** after any argument for further explanation.
+
+```bash
+> intelmqctl run file-output --help
+```
+
+Note that if another instance of the bot is running, only warning will be displayed.
+
+```bash
+> intelmqctl run file-output
+intelmqctl: Main instance of the bot is running in the background. You may want to launch: intelmqctl stop file-output
+```
+
+
+#### console
+
+If launched with **console** argument, you get a ```pdb``` live console; or ```ipdb``` or ```pudb``` consoles if they were previously installed (I.E. ```pip3 install ipdb --user```).
+
+```bash
+> intelmqctl run file-output console
+*** Using console ipdb. Please use 'self' to access to the bot instance properties. ***
+ipdb> self. ...
+```
+
+You may specify the desired console in the next argument.
+
+```bash
+> intelmqctl run file-output console pudb
+```
+
+#### message
+
+Operate directly with the input / output pipelines.
+
+If **get** is the parameter, you see the message that waits in the input (source or internal) queue. If the argument is **pop**, the message gets popped as well.
+
+```bash
+> intelmqctl run file-output message get
+file-output: Waiting for a message to get...
+{
+ "classification.type": "c&c",
+ "feed.url": "https://example.com",
+ "raw": "1233",
+ "source.ip": "1.2.3.4",
+ "time.observation": "2017-05-17T22:00:33+00:00",
+ "time.source": "2017-05-17T22:00:32+00:00"
+}
+```
+
+To send directly to the bot's ouput queue, just as it was sent by ```self.send_message()``` in bot's ```process()``` method, use the **send** argument.
+In our case of ```file-output```, it has no destionation queue so that nothing happens.
+
+```bash
+> intelmqctl run file-output message send '{"time.observation": "2017-05-17T22:00:33+00:00", "time.source": "2017-05-17T22:00:32+00:00"}'
+file-output: Bot has no destination queues.
+```
+
+Note, if you would like to know possible parameters of the message, put a wrong one – you will be prompted if you want to list all the current bot harmonization.
+
+#### process
+
+With no other arguments, bot\'s ```process()``` method will be run one time.
+
+```bash
+> intelmqctl run file-output process
+file-output: Bot is starting.
+file-output: Pipeline ready.
+file-output: Processing...
+file-output: Waiting for incoming message.
+file-output: Received message {'raw': '1234'}.
+```
+
+If run with **--dryrun|-d** flag, the message gets never really popped out from the source or internal pipeline, nor send to the output pipeline.
+Plus, you receive a note about the exact moment the message would get sent, or acknowledged.
+
+```bash
+> intelmqctl run file-output process -d
+file-output: * Dryrun only, no message will be really sent through.
+...
+file-output: DRYRUN: Message would be acknowledged now!
+```
+
+You may trick the bot to process a JSON instead of the Message in its pipeline with **--msg|-m** flag.
+
+```bash
+> intelmqctl run file-output process -m '{"source.ip":"1.2.3.4"}'
+file-output: * Message from cli will be used when processing.
+...
+```
+
 ### disable
 
 Sets the `enabled` flag in runtime.conf to `false`.

intelmq/bin/intelmqctl.py

@@ -15,6 +15,7 @@
  STARTUP_CONF_FILE, SYSTEM_CONF_FILE, VAR_RUN_PATH,
  BOTS_FILE)
 from intelmq.lib import utils
+from intelmq.lib.bot_debugger import BotDebugger
 from intelmq.lib.pipeline import PipelineFactory
 
 
@@ -117,33 +118,37 @@ def __init__(self, runtime_configuration, logger, controller):
  self.logger.error('Directory %s does not exist and cannot be '
  'created: %s.', self.PIDDIR, exc)
 
- def bot_run(self, bot_id):
+ def bot_run(self, bot_id, run_subcommand=None, console_type=None, message_action_kind=None, dryrun=None, msg=None):
  pid = self.__read_pidfile(bot_id)
- if pid:
- if self.__status_process(pid):
- log_bot_error('running', bot_id)
- return 'running'
- else:
- self.__remove_pidfile(bot_id)
+ if pid and self.__status_process(pid):
+ self.logger.warning("Main instance of the bot is running in the background and will be stopped; "
+ "when finished, we try to relaunch it again. "
+ "You may want to launch: 'intelmqctl stop {}' to prevent this message."
+ .format(bot_id))
+ paused = True
+ self.bot_stop(bot_id)
+ else:
+ paused = False
+
  log_bot_message('starting', bot_id)
  filename = self.PIDFILE.format(bot_id)
  with open(filename, 'w') as fp:
-  fp.write(str(os.getpid()))
+ fp.write(str(os.getpid()))
 
- bot_module = self.__runtime_configuration[bot_id]['module']
- module = importlib.import_module(bot_module)
- bot = getattr(module, 'BOT')
  try:
- instance = bot(bot_id)
- instance.start()
- except (Exception, KeyboardInterrupt) as exc:
- print('Bot failed: %s' % exc)
- retval = 1
+ BotDebugger(self.__runtime_configuration[bot_id], bot_id, run_subcommand,
+ console_type, dryrun, message_action_kind, msg)
+ retval = 0
+ except KeyboardInterrupt:
+ print('Keyboard interrupt.')
+ retval = 0
  except SystemExit as exc:
  print('Bot exited with code %s.' % exc)
  retval = exc
 
  self.__remove_pidfile(bot_id)
+ if paused:
+ self.bot_start(bot_id)
  return retval
 
  def bot_start(self, bot_id, getstatus=True):
@@ -305,10 +310,13 @@ def __init__(self, interactive: bool=False, return_type: str="python", quiet: bo
 
  Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
  EPILOG = '''
- intelmqctl [start|stop|restart|status|reload|run] bot-id
+ intelmqctl [start|stop|restart|status|reload] bot-id
  intelmqctl [start|stop|restart|status|reload]
  intelmqctl list [bots|queues]
  intelmqctl log bot-id [number-of-lines [log-level]]
+ intelmqctl run bot-id message [get|pop|send]
+ intelmqctl run bot-id process [--msg|--dryrun]
+ intelmqctl run bot-id console
  intelmqctl clear queue-id
  intelmqctl check
 
@@ -321,8 +329,14 @@ def __init__(self, interactive: bool=False, return_type: str="python", quiet: bo
 Get status of a bot:
  intelmqctl status bot-id
 
-Run a bot directly (blocking) for debugging purpose:
+Run a bot directly for debugging purpose and temporarily leverage the logging level to DEBUG:
  intelmqctl run bot-id
+Get a pdb (or ipdb if installed) live console.
+ intelmqctl run bot-id console
+See the message that waits in the input queue.
+ intelmqctl run bot-id message get
+See additional help for further explanation.
+ intelmqctl run bot-id --help
 
 Starting the botnet (all bots):
  intelmqctl start
@@ -434,6 +448,30 @@ def __init__(self, interactive: bool=False, return_type: str="python", quiet: bo
  parser_run = subparsers.add_parser('run', help='Run a bot interactively')
  parser_run.add_argument('bot_id',
  choices=self.runtime_configuration.keys())
+ parser_run_subparsers = parser_run.add_subparsers(title='run-subcommands')
+
+ parser_run_console = parser_run_subparsers.add_parser('console', help='Get a ipdb live console.')
+ parser_run_console.add_argument('console_type', nargs='?',
+ help='You may specify which console should be run. Default is ipdb (if installed)'
+ ' or pudb (if installed) or pdb but you may want to use another one.')
+ parser_run_console.set_defaults(run_subcommand="console")
+
+ parser_run_message = parser_run_subparsers.add_parser('message',
+ help='Debug bot\'s pipelines. Get the message in the'
+ ' input pipeline, pop it (cut it) and display it, or'
+ ' send the message directly to bot\'s output pipeline.')
+ parser_run_message.add_argument('message_action_kind', choices=["get", "pop", "send"])
+ parser_run_message.add_argument('msg', nargs='?', help='If send was chosen, put here the message in JSON.')
+ parser_run_message.set_defaults(run_subcommand="message")
+
+ parser_run_process = parser_run_subparsers.add_parser('process', help='Single run of bot\'s process() method.')
+ parser_run_process.add_argument('--dryrun', '-d', action='store_true',
+ help='Never really pop the message from the input pipeline '
+ 'nor send to output pipeline.')
+ parser_run_process.add_argument('--msg', '-m',
+ help='Trick the bot to process this JSON '
+ 'instead of the Message in its pipeline.')
+ parser_run_process.set_defaults(run_subcommand="process")
  parser_run.set_defaults(func=self.bot_run)
 
  parser_check = subparsers.add_parser('check',
@@ -518,8 +556,8 @@ def run(self):
  elif results == 'error':
  return 1
 
- def bot_run(self, bot_id):
- return self.bot_process_manager.bot_run(bot_id)
+ def bot_run(self, **kwargs):
+ return self.bot_process_manager.bot_run(**kwargs)
 
  def bot_start(self, bot_id, getstatus=True):
  if bot_id is None: