From 6fe8debebab0592dd80ca4ce5abd9a0311f0d84e Mon Sep 17 00:00:00 2001 From: computergeek125 Date: Sat, 25 Sep 2021 12:25:27 -0500 Subject: [PATCH 1/6] Set TCP timeout to a more reasonable value --- app/classes/shared/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 0bb18fa4..6b37d6fc 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -90,6 +90,7 @@ class Helpers: try: host_public = get('https://api.ipify.org').text sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(10.0) result = sock.connect_ex((host_public ,server_port)) sock.close() if result == 0: From 4f320e69a5561c4d8081db3b7745a3e76d2bb316 Mon Sep 17 00:00:00 2001 From: computergeek125 Date: Sat, 25 Sep 2021 14:29:03 -0500 Subject: [PATCH 2/6] Added string to array command parser --- app/classes/shared/helpers.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 6b37d6fc..730881b4 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -35,6 +35,11 @@ except ModuleNotFoundError as e: sys.exit(1) class Helpers: + allowed_quotes = [ + "\"", + "'", + "`" + ] def __init__(self): self.root_dir = os.path.abspath(os.path.curdir) @@ -100,6 +105,37 @@ class Helpers: except Exception as err: return False + @staticmethod + def cmdparse(cmd_in): + # Parse a string into arguments + cmd_out = [] # "argv" output array + ci = -1 # command index - pointer to the argument we're building in cmd_out + np = True # whether we're creating a new argument/parameter + esc = False # whether an escape character was encountered + stch = None # if we're dealing with a quote, save the quote type here. Nested quotes to be dealt with by the command + for c in cmd_in: # for character in string + if np == True: # if set, begin a new argument and increment the command index. Continue the loop. + if c == ' ': + continue + else: + ci += 1 + cmd_out.append("") + np = False + if esc: # if we encountered an escape character on the last loop, append this char regardless of what it is + cmd_out[ci] += c + esc = False + else: + if c == '\\': # if the current character is an escape character, set the esc flag and continue to next loop + esc = True + elif c == ' ' and stch is None: # if we encounter a space and are not dealing with a quote, set the new argument flag and continue to next loop + np = True + elif c == stch: # if we encounter the character that matches our start quote, end the quote and continue to next loop + stch = None + elif stch is None and (c in Helpers.allowed_quotes): # if we're not in the middle of a quote and we get a quotable character, start a quote and proceed to the next loop + stch = c + else: # else, just store the character in the current arg + cmd_out[ci] += c + return cmd_out def check_for_old_logs(self, db_helper): servers = db_helper.get_all_defined_servers() From 250b68ae51ebd81fa5db4b32089e662c8a33e04a Mon Sep 17 00:00:00 2001 From: computergeek125 Date: Sat, 25 Sep 2021 14:29:28 -0500 Subject: [PATCH 3/6] Protype subprocess management --- app/classes/minecraft/stats.py | 11 +-- app/classes/shared/controller.py | 29 ------- app/classes/shared/server.py | 145 +++++++++++++++---------------- app/classes/web/ajax_handler.py | 9 +- 4 files changed, 80 insertions(+), 114 deletions(-) diff --git a/app/classes/minecraft/stats.py b/app/classes/minecraft/stats.py index a173c2df..382cd173 100644 --- a/app/classes/minecraft/stats.py +++ b/app/classes/minecraft/stats.py @@ -45,15 +45,16 @@ class Stats: return data @staticmethod - def _get_process_stats(process_pid: int): - if process_pid is None: + def _get_process_stats(process): + if process is None: process_stats = { 'cpu_usage': 0, 'memory_usage': 0, 'mem_percentage': 0 } return process_stats - + else: + process_pid = process.pid try: p = psutil.Process(process_pid) dummy = p.cpu_percent() @@ -217,7 +218,7 @@ class Stats: world_path = os.path.join(server_data.get('path', None), world_name) # process stats - p_stats = self._get_process_stats(server_obj.PID) + p_stats = self._get_process_stats(server_obj.process) # TODO: search server properties file for possible override of 127.0.0.1 internal_ip = server['server_ip'] @@ -275,7 +276,7 @@ class Stats: world_path = os.path.join(server_data.get('path', None), world_name) # process stats - p_stats = self._get_process_stats(server_obj.PID) + p_stats = self._get_process_stats(server_obj.process) # TODO: search server properties file for possible override of 127.0.0.1 internal_ip = server['server_ip'] diff --git a/app/classes/shared/controller.py b/app/classes/shared/controller.py index 8e72f36a..d8d883d8 100644 --- a/app/classes/shared/controller.py +++ b/app/classes/shared/controller.py @@ -247,38 +247,9 @@ class Controller: console.info("All Servers Stopped") def stop_server(self, server_id): - # get object - svr_obj = self.get_server_obj(server_id) - svr_data = self.get_server_data(server_id) - server_name = svr_data['server_name'] - - running = svr_obj.check_running() - # issue the stop command svr_obj.stop_threaded_server() - # while it's running, we wait - x = 0 - while running: - logger.info("Server {} is still running - waiting 2s to see if it stops".format(server_name)) - console.info("Server {} is still running - waiting 2s to see if it stops".format(server_name)) - running = svr_obj.check_running() - - # let's keep track of how long this is going on... - x = x + 1 - - # if we have been waiting more than 120 seconds. let's just kill the pid - if x >= 60: - logger.error("Server {} is taking way too long to stop. Killing this process".format(server_name)) - console.error("Server {} is taking way too long to stop. Killing this process".format(server_name)) - - svr_obj.killpid(svr_obj.PID) - running = False - - # if we killed the server, let's clean up the object - if not running: - svr_obj.cleanup_server_object() - def create_jar_server(self, server: str, version: str, name: str, min_mem: int, max_mem: int, port: int): server_id = helper.create_uuid() server_dir = os.path.join(helper.servers_dir, server_id) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 36fe5852..74770bd8 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -3,15 +3,13 @@ import sys import re import json import time -import psutil -import pexpect import datetime import threading -import schedule import logging.config import zipfile from threading import Thread import shutil +import subprocess import zlib import html @@ -26,7 +24,9 @@ logger = logging.getLogger(__name__) try: - import pexpect + import psutil + #import pexpect + import schedule except ModuleNotFoundError as e: logger.critical("Import Error: Unable to load {} module".format(e.name), exc_info=True) @@ -37,8 +37,8 @@ except ModuleNotFoundError as e: class ServerOutBuf: lines = {} - def __init__(self, p, server_id): - self.p = p + def __init__(self, proc, server_id): + self.proc = proc self.server_id = str(server_id) # Buffers text for virtual_terminal_lines config number of lines self.max_lines = helper.get_setting('virtual_terminal_lines') @@ -46,10 +46,12 @@ class ServerOutBuf: ServerOutBuf.lines[self.server_id] = [] def check(self): - while self.p.isalive(): - char = self.p.read(1) + while self.proc.poll() is None: + char = self.proc.stdout.read(1).decode('utf-8') + # TODO: we may want to benchmark reading in blocks and userspace processing it later, reads are kind of expensive as a syscall if char == os.linesep: ServerOutBuf.lines[self.server_id].append(self.line_buffer) + self.new_line_handler(self.line_buffer) self.line_buffer = '' # Limit list length to self.max_lines: @@ -84,7 +86,6 @@ class Server: # holders for our process self.process = None self.line = False - self.PID = None self.start_time = None self.server_command = None self.server_path = None @@ -141,7 +142,7 @@ class Server: def setup_server_run_command(self): # configure the server server_exec_path = self.settings['executable'] - self.server_command = self.settings['execution_command'] + self.server_command = helper.cmdparse(self.settings['execution_command']) self.server_path = self.settings['path'] # let's do some quick checking to make sure things actually exists @@ -178,13 +179,15 @@ class Server: if os.name == "nt": logger.info("Windows Detected") + creationflags=subprocess.CREATE_NEW_CONSOLE self.server_command = self.server_command.replace('\\', '/') else: - logger.info("Linux Detected") + logger.info("Unix Detected") + creationflags=None logger.info("Starting server in {p} with command: {c}".format(p=self.server_path, c=self.server_command)) try: - self.process = pexpect.spawn(self.server_command, cwd=self.server_path, timeout=None, encoding=None) + self.process = subprocess.Popen(self.server_command, cwd=self.server_path, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except Exception as ex: msg = "Server {} failed to start with error code: {}".format(self.name, ex) logger.error(msg) @@ -205,9 +208,7 @@ class Server: websocket_helper.broadcast('send_start_error', { 'error': translation.translate('error', 'internet') }) - db_helper.set_waiting_start(self.server_id, False) - self.process = pexpect.spawn(self.server_command, cwd=self.server_path, timeout=None, encoding='utf-8') out_buf = ServerOutBuf(self.process, self.server_id) logger.debug('Starting virtual terminal listener for server {}'.format(self.name)) @@ -217,15 +218,14 @@ class Server: self.start_time = str(datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')) - if psutil.pid_exists(self.process.pid): - self.PID = self.process.pid - logger.info("Server {} running with PID {}".format(self.name, self.PID)) - console.info("Server {} running with PID {}".format(self.name, self.PID)) + if self.process.poll() is None: + logger.info("Server {} running with PID {}".format(self.name, self.process.pid)) + console.info("Server {} running with PID {}".format(self.name, self.process.pid)) self.is_crashed = False self.stats.record_stats() else: - logger.warning("Server PID {} died right after starting - is this a server config issue?".format(self.PID)) - console.warning("Server PID {} died right after starting - is this a server config issue?".format(self.PID)) + logger.warning("Server PID {} died right after starting - is this a server config issue?".format(self.process.pid)) + console.warning("Server PID {} died right after starting - is this a server config issue?".format(self.process.pid)) if self.settings['crash_detection']: logger.info("Server {} has crash detection enabled - starting watcher task".format(self.name)) @@ -242,33 +242,36 @@ class Server: def stop_server(self): if self.settings['stop_command']: self.send_command(self.settings['stop_command']) - - running = self.check_running() - x = 0 - - # caching the name and pid number - server_name = self.name - server_pid = self.PID - - while running: - x = x+1 - logger.info("Server {} is still running - waiting 2s to see if it stops".format(server_name)) - console.info("Server {} is still running - waiting 2s to see if it stops".format(server_name)) - console.info("Server has {} seconds to respond before we force it down".format(int(60-(x*2)))) - running = self.check_running() - time.sleep(2) - - # if we haven't closed in 60 seconds, let's just slam down on the PID - if x >= 30: - logger.info("Server {} is still running - Forcing the process down".format(server_name)) - console.info("Server {} is still running - Forcing the process down".format(server_name)) - self.process.terminate(force=True) - - logger.info("Stopped Server {} with PID {}".format(server_name, server_pid)) - console.info("Stopped Server {} with PID {}".format(server_name, server_pid)) - else: - self.process.terminate(force=True) + #windows will need to be handled separately for Ctrl+C + self.process.terinate() + running = self.check_running() + if not running: + logger.info("Can't stop server {} if it's not running".format(self.name)) + console.info("Can't stop server {} if it's not running".format(self.name)) + return + x = 0 + + # caching the name and pid number + server_name = self.name + server_pid = self.process.pid + + while running: + x = x+1 + logstr = "Server {} is still running - waiting 2s to see if it stops ({} seconds until force close)".format(server_name, int(60-(x*2))) + logger.info(logstr) + console.info(logstr) + running = self.check_running() + time.sleep(2) + + # if we haven't closed in 60 seconds, let's just slam down on the PID + if x >= 30: + logger.info("Server {} is still running - Forcing the process down".format(server_name)) + console.info("Server {} is still running - Forcing the process down".format(server_name)) + self.kill() + + logger.info("Stopped Server {} with PID {}".format(server_name, server_pid)) + console.info("Stopped Server {} with PID {}".format(server_name, server_pid)) # massive resetting of variables self.cleanup_server_object() @@ -286,7 +289,6 @@ class Server: self.run_threaded_server() def cleanup_server_object(self): - self.PID = None self.start_time = None self.restart_count = 0 self.is_crashed = False @@ -294,35 +296,27 @@ class Server: self.process = None def check_running(self): - running = False # if process is None, we never tried to start - if self.PID is None: - return running - - try: - alive = self.process.isalive() - if type(alive) is not bool: - self.last_rc = alive - running = False - else: - running = alive - - except Exception as e: - logger.error("Unable to find if server PID exists: {}".format(self.PID), exc_info=True) - pass - - return running + if self.process is None: + return False + poll = self.process.poll() + if poll is None: + return True + else: + self.last_rc = poll + return False def send_command(self, command): - + console.info("COMMAND TIME: {}".format(command)) if not self.check_running() and command.lower() != 'start': logger.warning("Server not running, unable to send command \"{}\"".format(command)) return False - logger.debug("Sending command {} to server via pexpect".format(command)) + logger.debug("Sending command {} to server".format(command)) # send it - self.process.send(command + '\n') + self.process.stdin.write("{}\n".format(command).encode('utf-8')) + self.process.stdin.flush() def crash_detected(self, name): @@ -344,18 +338,18 @@ class Server: "The server {} has crashed, crash detection is disabled and it will not be restarted".format(name)) return False - def killpid(self, pid): - logger.info("Terminating PID {} and all child processes".format(pid)) - process = psutil.Process(pid) + def kill(self): + logger.info("Terminating server {} and all child processes".format(self.server_id)) + process = psutil.Process(self.process.pid) # for every sub process... for proc in process.children(recursive=True): # kill all the child processes - it sounds too wrong saying kill all the children (kevdagoat: lol!) - logger.info("Sending SIGKILL to PID {}".format(proc.name)) + logger.info("Sending SIGKILL to server {}".format(proc.name)) proc.kill() # kill the main process we are after logger.info('Sending SIGKILL to parent') - process.kill() + self.process.kill() def get_start_time(self): if self.check_running(): @@ -364,7 +358,10 @@ class Server: return False def get_pid(self): - return self.PID + if self.process is not None: + return self.process.pid + else: + return None def detect_crash(self): @@ -476,7 +473,7 @@ class Server: #checks if server is running. Calls shutdown if it is running. if self.check_running(): wasStarted = True - logger.info("Server with PID {} is running. Sending shutdown command".format(self.PID)) + logger.info("Server with PID {} is running. Sending shutdown command".format(self.process.pid)) self.stop_threaded_server() else: wasStarted = False diff --git a/app/classes/web/ajax_handler.py b/app/classes/web/ajax_handler.py index 55bc5bf1..3cb8c1c1 100644 --- a/app/classes/web/ajax_handler.py +++ b/app/classes/web/ajax_handler.py @@ -203,12 +203,9 @@ class AjaxHandler(BaseHandler): elif page == "kill": server_id = self.get_argument('id', None) svr = self.controller.get_server_obj(server_id) - if svr.get_pid(): - try: - svr.killpid(svr.get_pid()) - except Exception as e: - logger.error("Could not find PID for requested termsig. Full error: {}".format(e)) - else: + try: + svr.kill() + except Exception as e: logger.error("Could not find PID for requested termsig. Full error: {}".format(e)) return From 82f4661044d23f305003e8b0e112bcc09c5e464e Mon Sep 17 00:00:00 2001 From: computergeek125 Date: Sat, 6 Nov 2021 11:27:15 -0500 Subject: [PATCH 4/6] Fixed db_helper issue --- app/classes/shared/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 673aeff3..36328ce7 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -211,7 +211,7 @@ class Server: websocket_helper.broadcast('send_start_error', { 'error': translation.translate('error', 'internet', user_lang) }) - db_helper.set_waiting_start(self.server_id, False) + servers_helper.set_waiting_start(self.server_id, False) out_buf = ServerOutBuf(self.process, self.server_id) logger.debug('Starting virtual terminal listener for server {}'.format(self.name)) From 9720929e33575ee44237819ec559faa32b6d7da5 Mon Sep 17 00:00:00 2001 From: computergeek125 Date: Sat, 6 Nov 2021 12:06:26 -0500 Subject: [PATCH 5/6] Resolved missing stdout when java process exits quickly --- app/classes/shared/server.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index 36328ce7..fb582ae7 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -46,20 +46,28 @@ class ServerOutBuf: self.line_buffer = '' ServerOutBuf.lines[self.server_id] = [] - def check(self): - while self.proc.poll() is None: - char = self.proc.stdout.read(1).decode('utf-8') - # TODO: we may want to benchmark reading in blocks and userspace processing it later, reads are kind of expensive as a syscall - if char == os.linesep: - ServerOutBuf.lines[self.server_id].append(self.line_buffer) + def process_byte(self, char): + if char == os.linesep: + ServerOutBuf.lines[self.server_id].append(self.line_buffer) - self.new_line_handler(self.line_buffer) - self.line_buffer = '' - # Limit list length to self.max_lines: - if len(ServerOutBuf.lines[self.server_id]) > self.max_lines: - ServerOutBuf.lines[self.server_id].pop(0) + self.new_line_handler(self.line_buffer) + self.line_buffer = '' + # Limit list length to self.max_lines: + if len(ServerOutBuf.lines[self.server_id]) > self.max_lines: + ServerOutBuf.lines[self.server_id].pop(0) + else: + self.line_buffer += char + + def check(self): + while True: + if self.proc.poll() is None: + char = self.proc.stdout.read(1).decode('utf-8') + # TODO: we may want to benchmark reading in blocks and userspace processing it later, reads are kind of expensive as a syscall + self.process_byte(char) else: - self.line_buffer += char + flush = self.proc.stdout.read().decode('utf-8') + for char in flush: + self.process_byte(char) def new_line_handler(self, new_line): new_line = re.sub('(\033\\[(0;)?[0-9]*[A-z]?(;[0-9])?m?)|(> )', '', new_line) From 64ec33ff520a48fed4b0c89780e84b2a7181345a Mon Sep 17 00:00:00 2001 From: computergeek125 Date: Sun, 14 Nov 2021 18:17:23 -0600 Subject: [PATCH 6/6] Resolved subprocess issues with windows - Repaired introduced bug where backslashes in the far path weren't handled properly - Resolved latent subprocess issue where stdout would fail to send websocket notifications when a a windows \r\n was emitted --- app/classes/shared/helpers.py | 2 ++ app/classes/shared/server.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/classes/shared/helpers.py b/app/classes/shared/helpers.py index 7dc15d8e..f6442ac1 100644 --- a/app/classes/shared/helpers.py +++ b/app/classes/shared/helpers.py @@ -121,6 +121,8 @@ class Helpers: cmd_out.append("") np = False if esc: # if we encountered an escape character on the last loop, append this char regardless of what it is + if c not in Helpers.allowed_quotes: + cmd_out[ci] += '\\' cmd_out[ci] += c esc = False else: diff --git a/app/classes/shared/server.py b/app/classes/shared/server.py index fb582ae7..2a433a65 100644 --- a/app/classes/shared/server.py +++ b/app/classes/shared/server.py @@ -45,9 +45,17 @@ class ServerOutBuf: self.max_lines = helper.get_setting('virtual_terminal_lines') self.line_buffer = '' ServerOutBuf.lines[self.server_id] = [] + self.lsi = 0 def process_byte(self, char): - if char == os.linesep: + if char == os.linesep[self.lsi]: + self.lsi += 1 + else: + self.lsi = 0 + self.line_buffer += char + + if self.lsi >= len(os.linesep): + self.lsi = 0 ServerOutBuf.lines[self.server_id].append(self.line_buffer) self.new_line_handler(self.line_buffer) @@ -55,8 +63,6 @@ class ServerOutBuf: # Limit list length to self.max_lines: if len(ServerOutBuf.lines[self.server_id]) > self.max_lines: ServerOutBuf.lines[self.server_id].pop(0) - else: - self.line_buffer += char def check(self): while True: @@ -189,7 +195,6 @@ class Server: if os.name == "nt": logger.info("Windows Detected") creationflags=subprocess.CREATE_NEW_CONSOLE - self.server_command = self.server_command.replace('\\', '/') else: logger.info("Unix Detected") creationflags=None