django+tornado实现实时查看远程日志的方法

2019-11-25 12:01:35

字体：大中小

来源：转载

供稿：网友

大致思路：

1.利用tornado提供的websocket功能与浏览器建立长连接，读取实时日志并输出到浏览器

2.写一个实时读取日志的脚本，利用saltstack远程执行，并把实时日志发往redis中。

3.tornado读取redis中的信息，发往浏览器。

此过程用到了redis的发布和订阅功能。

先看一下tornado中是如何处理的：

import osimport sysimport tornado.websocketimport tornado.webimport tornado.ioloopimport redisimport salt.clientfrom tornado import genfrom tornado.escape import to_unicodefrom logs.utility import get_last_linesfrom logs import settingsclass SubWebSocket(tornado.websocket.WebSocketHandler): """ 此handler处理远程日志查看 """ def open(self, *args, **kwargs):  print("opened") @gen.coroutine def on_message(self, message):  # 主机名，要查看的日志路径，运行脚本的命令这些信息从浏览器传过来  hostname, log_path, cmd = message.split("||")  local = salt.client.LocalClient()  r = redis.StrictRedis(host=settings.REDIS_HOST, port=settings.REDIS_PORT,        password=settings.REDIS_PASSWD, db=5)  # 订阅频道，服务器和日志路径确定一个频道  key = settings.LOG_KEY.format(server=hostname.strip(), log_path=log_path.strip())  channel = r.pubsub()  channel.subscribe(key)  # 异步方式执行命令，远程运行脚本  local.cmd_async(hostname, "cmd.run", [cmd])  try:   while True:    data = channel.get_message()    if not data:     # 如果读取不到消息，间隔一定时间，避免无谓的CPU消耗     yield gen.sleep(0.05)     continue    if data["type"] == "message":     line = format_line(data["data"])     self.write_message(line)  except tornado.websocket.WebSocketClosedError:   self.close() def on_close(self):  global FLAG  FLAG = False  print("closed")def format_line(line): line = to_unicode(line) if "INFO" in line:  color = "#46A3FF" elif "WARN" in line:  color = "#FFFF37" elif "ERROR" in line:  color = "red" elif "CRITICAL" in line:  color = "red" else:  color = "#FFFFFF" return "<span style='color:{}'>{}</span>".format(color, line)class EchoWebSocket(tornado.websocket.WebSocketHandler): def open(self):  print("WebSocket opened") @gen.coroutine def on_message(self, message):  log = message  print "log file: ", log  try:   with open(log, 'r') as f:    for line in get_last_lines(f):     line1 = format_line(line)     self.write_message(line1)    while True:     line = f.readline()     if not line:      yield gen.sleep(0.05)      continue     self.write_message(format_line(line.strip()))  except tornado.websocket.WebSocketClosedError as e:   print e   self.close() # def check_origin(self, origin): #  print origin, self.request.headers.get("Host") #  # super(EchoWebSocket, self).check_origin() #  return True def on_close(self):  print("WebSocket closed")class Application(tornado.web.Application): def __init__(self):  handlers = [   (r'/log/', MainHandler), # 提供浏览页面，页面中的JS与服务器建立连接   (r'/log/local', EchoWebSocket), # 处理本地日志实时查看，比较简单   (r'/log/remote', SubWebSocket), # 处理远程日志实时查看，稍微复杂  ]  settings = {   "debug": True,   "template_path": os.path.join(os.path.dirname(__file__), "templates"),   "static_path": os.path.join(os.path.dirname(__file__), "static"),  }  super(Application, self).__init__(handlers, **settings)class MainHandler(tornado.web.RequestHandler): def get(self):  # 要查看的日志路径  log = self.get_argument("log", None)  # hostname实际上是saltstack中这台机器对应的minion id  hostname = self.get_argument("hostname", None)  # 本地日志还是远程日志  type = self.get_argument("type", "local")  # 运行读取实时日志的脚本，参数比较多，后面会有  cmd = self.get_argument("cmd", "")  context = {   "log": log,   "hostname": hostname,   "type": type,   "cmd": cmd,  }  self.render("index.html", **context)

配置文件中主要记录了redis服务器的地址等信息

# encoding: utf-8LOG_KEY = "logs:{server}:{log_path}"LOG_NAME = "catalina.out"TAIL_LINE_NUM = 20REDIS_HOST = "127.0.0.1"REDIS_PORT = "6379"REDIS_PASSWD = NoneREDIS_EXPIRE = 300try: from local_settings import *except ImportError: pass

index.html的内容如下：

<html><head><link href="{{ static_url('public/css/public.css') }}" rel="external nofollow" rel="stylesheet" /><link href="{{ static_url('kylin/css/style.css') }}" rel="external nofollow" rel="stylesheet" /></head><body style="background:#000000"><div style="margin-left:10px;"> <pre id="id-content"> </pre> <div id="id-bottom"></div> <input type="hidden" id="id-log" value="{{ log }}" /> <input type="hidden" id="id-type" value="{{ type }}" /> <input type="hidden" id="id-hostname" value="{{ hostname }}" /> <input type="hidden" id="id-cmd" value="{{ cmd }}" /> <div class="btns btns_big">  <button type="button" class="query_btn cancle" id="id-stop">Stop</button>  <button type="button" class="query_btn commit" id="id-start">Start</button> </div></div><script type="text/javascript" src="{{ static_url('js/jquery-1.11.3.min.js') }}"></script><script type="text/javascript"> var log_name = $("#id-log").val(); var type = $("#id-type").val(); var hostname = $("#id-hostname").val(); var cmd = $("#id-cmd").val(); // 初始化websocket对象 var ws = new WebSocket("ws://{{ request.host }}/log/" + type); ws.onopen = function(){  if (type === "local"){   ws.send(log_name);  } else {   // 建立连接后把相关信息发往服务器，对应上面的SubWebSocket   ws.send(hostname + "||" + log_name + "||" + cmd);  } }; var get_message = function(evt){  $("#id-content").append(evt.data + "/n");  document.getElementById("id-bottom").scrollIntoView() }; ws.onmessage = get_message; // 两个按钮控制日志的输出，如果看到需要的日志信息，可以暂停日志的输出， // 之后可以继续启动日志的输出 $("#id-stop").click(function(){  ws.onmessage = function(){}; }) $("#id-start").click(function(){  ws.onmessage = get_message; })</script></body></html>

这个tornado仅仅是提供了实时日志的服务，实际项目使用的是django，django中要做的其实很简单，提供log_name,hostname,type,cmd等四个参数。

下面看一个实例：

class LogView(KylinView): # 实时读取日志的脚本，事先使用saltstack批量传到各台服务器上 client_path = "/tmp/logtail.py" def get(self, request):  minion_id = request.GET.get("minion_id")  context = {   "minion_id": minion_id,   "tail_log_url": settings.TAIL_LOG_URL,  }  return render(request, "cmdb/log_view.html", context) def post(self, request):  minion_id = request.POST.get("minion_id")  log_path = request.POST.get("log_path")  if not log_path:   return JsonResponse({"success": False, "message": "请填写日志路径"})  try:   # 制定一开始读取的行数   line_count = request.POST.get("line_count")  except (TypeError, ValueError):   return JsonResponse({"success": False, "message": "请输入正确的行数"})  local = salt.client.LocalClient()  # 确保saltstack能连通并且日志文件存在  ret = local.cmd(minion_id, "file.file_exists", [log_path])  if minion_id not in ret:   return JsonResponse({"success": False, "message": "服务器无法连通"})  if not ret[minion_id]:   return JsonResponse({"success": False, "message": "日志文件不存在"})  # 组成命令的各个参数，redis信息需要和tornado配置文件中的redis信息一致  cmd = "{} {} {} {} {} {} {} {}".format(   settings.PYTHON_BIN, self.client_path, minion_id, log_path, line_count, settings.REDIS_HOST,   settings.REDIS_PORT, settings.REDIS_PASSWD)  # settings.TAIL_LOG_URL是tornado中MainHandler对应的url,把其它几个  # 参数组合成最终的URL，直接访问这个URL就可以在浏览器中实时读取日志了。  url = "{}?type=remote&log={}&hostname={}&cmd={}".format(   settings.TAIL_LOG_URL, log_path, minion_id, cmd)  # 这一步的操作确保同一个日志文件只有一个脚本在读取，避免日志信息重复，这一步  # 也很重要，必不可少  local.cmd(minion_id, "cmd.run",     ["kill `ps aux|grep logtail.py|grep %s|grep -v grep|awk '{print $2}'`" % (log_path,)])  return JsonResponse({"success": True, "url": url})

下面来看看logtail.py的实现：

# encoding: utf-8from __future__ import unicode_literals, divisionimport mathimport timeimport sysimport socketimport signalimport redisFLAG = Truedef get_last_lines(f, num=10): """读取文件的最后几行 """ size = 1000 try:  f.seek(-size, 2) except IOError: # 文件内容不足size  f.seek(0)  return f.readlines()[-num:] data = f.read() lines = data.splitlines() n = len(lines) while n < num:  size *= int(math.ceil(num / n))  try:   f.seek(-size, 2)  except IOError:   f.seek(0)   return f.readlines()[-num:]  data = f.read()  lines = data.splitlines()  n = len(lines) return lines[-num:]def process_line(r, channel, line): r.publish(channel, line.strip())def sig_handler(signum, frame): global FLAG FLAG = False# 收到退出信号后，以比较优雅的方式终止脚本signal.signal(signal.SIGTERM, sig_handler)# 为了避免日志输出过多，浏览器承受不住，设置5分钟后脚本自动停止signal.signal(signal.SIGALRM, sig_handler)signal.alarm(300)def get_hostname(): return socket.gethostname()def force_str(s): if isinstance(s, unicode):  s = s.encode("utf-8") return sdef tail(): password = sys.argv[6] if password == "None":  password = None r = redis.StrictRedis(host=sys.argv[4], port=sys.argv[5], password=password, db=5) log_path = sys.argv[2] line_count = int(sys.argv[3]) # 往redis频道发送实时日志 channel = "logs:{hostname}:{log_path}".format(hostname=sys.argv[1], log_path=log_path) with open(log_path, 'r') as f:  last_lines = get_last_lines(f, line_count)  for line in last_lines:   process_line(r, channel, force_str(line))  try:   while FLAG: # 通过信号控制这个变量，实现优雅退出循环    line = f.readline()    if not line:     time.sleep(0.05)     continue    process_line(r, channel, line)  except KeyboardInterrupt:   pass print("Exiting...")if __name__ == "__main__": if len(sys.argv) < 6:  print "Usage: %s minion_id log_path host port redis_pass"  exit(1) tail()

到此为止，整个实时读取远程日志的流程就讲完了。

github: https://github.com/tuxinhang1989/logs

以上这篇django+tornado实现实时查看远程日志的方法就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持武林网。

上一篇：Python利用requests模块下载图片实例代码

下一篇：Django结合ajax进行页面实时更新的例子