MY FIRST “WATCH DOG” PROJECT
•
Background: I updated my virtual environments packages such as openai, langchain using pip
◦
Suddenly, all the service using the backend application running on my server downed.
◦
But, it took 2hours for me to realize the situation and my supervisor coudn’t demo out service to whom we will coporate…
◦
As u already guess, after updating the packages, some crushes occured…
◦
So, i need to make the watch dog which let me know if out service, especially backend service got downed.
The below is the code i wrote.
•
using the os.kill function, the process are monitoring the service process and run the backend application again.
import os
import time
import sys
import smtplib # SMTP 사용을 위한 모듈
import re # Regular Expression을 활용하기 위한 모듈
from email.mime.multipart import MIMEMultipart # 메일의 Data 영역의 메시지를 만드는 모듈
from email.mime.text import MIMEText # 메일의 본문 내용을 만드는 모듈
from email.mime.image import MIMEImage # 메일의 이미지 파일을 base64 형식으로 변환하기 위한 모듈
def is_process_running(pid):
"""Check if a process with the given PID is currently running."""
try:
os.kill(pid, 0)
except OSError:
return False
else:
return True
def monitor_process(pid):
"""Monitor the process with the given PID and notify when it stops running."""
while True:
if not is_process_running(pid):
print(f"Process {pid} has stopped running.")
# smpt 서버와 연결
gmail_smtp = "smtp.gmail.com" # gmail smtp 주소
gmail_port = 465 # gmail smtp 포트번호. 고정(변경 불가)
smtp = smtplib.SMTP_SSL(gmail_smtp, gmail_port)
# 로그인
my_account = "[CONFIDENTIAL]"
my_password = "[CONFIDENTIAL]"
smtp.login(my_account, my_password)
# 메일을 받을 계정
to_mail = "[CONFIDENTIAL]"
# 메일 기본 정보 설정
msg = MIMEMultipart()
msg["Subject"] = f""[CONFIDENTIAL]"" # 메일 제목
msg["From"] = my_account
msg["To"] = to_mail
# 메일 본문 내용
content = "[CONFIDENTIAL]"
content_part = MIMEText(content, "plain")
msg.attach(content_part)
# 받는 메일 유효성 검사 거친 후 메일 전송
sendEmail(to_mail, smtp, my_account, to_mail, msg)
# smtp 서버 연결 해제
smtp.quit()
#다시 실행
print(os.system("nohup python run.py > output.log 2>&1 &"))
break
else:
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print(f"Process {pid} is running at {now}")
time.sleep(30) # Check every second
def sendEmail(addr, smtp: smtplib.SMTP_SSL, my_account, to_mail, msg: MIMEMultipart):
reg = "^[a-zA-Z0-9.+_-]+@[a-zA-Z0-9]+\.[a-zA-Z]{2,3}$" # 유효성 검사를 위한 정규표현식
if re.match(reg, addr):
smtp.sendmail(my_account, to_mail, msg.as_string())
print("정상적으로 메일이 발송되었습니다.")
else:
print("받으실 메일 주소를 정확히 입력하십시오.")
# Replace with the PID of the process you want to monitor
if __name__ == '__main__':
pid_to_monitor = 0
try:
pid_to_monitor = int(str(sys.argv[1]))
except:
print("ERROR:",pid_to_monitor,"is not a valid PID")
pid_to_monitor = "[CONFIDENTIAL]"
print("Monitoring process with PID", pid_to_monitor)
monitor_process(pid_to_monitor)
Python
복사