| Server IP : 199.250.214.225 / Your IP : 18.226.94.138 Web Server : Apache System : Linux vps64074.inmotionhosting.com 3.10.0-1160.105.1.vz7.214.3 #1 SMP Tue Jan 9 19:45:01 MSK 2024 x86_64 User : nicngo5 ( 1001) PHP Version : 7.4.33 Disable Function : exec,passthru,shell_exec,system MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : ON | Sudo : ON | Pkexec : OFF Directory : /opt/dedrads/ |
Upload File : |
#!/opt/imh-python/bin/python3
'''
script to resolve known issues with new vps/dedi
and improve thier default configurations
it is to be ran with -c on provision to alert of issues via STR/stdout
-c is fast, but without flag does all the stuff that is slow (~30s)
will not send STRs unless the -s flag is used
Author: MathewL
'''
import shlex
import subprocess as sp
import os
import sys
import re
import math
import shutil
import argparse
import socket
import smtplib
class FirstSetup:
def __init__(self):
self.str_msg = ''
self.run_log = ''
self.get_args()
self.is_a_vps()
self.get_mem_total()
self.host_name = socket.gethostname()
def get_args(self):
'''
argz
'''
parser = argparse.ArgumentParser()
parser.add_argument(
'-s',
'--send-str',
action='store_true',
default=False,
help=('Silence most output, sends STR to T2S for errors.\n'),
)
parser.add_argument(
'-n',
'--no-ea4',
action='store_true',
default=False,
help=('Skips installing missing apache modules\n'),
)
parser.add_argument(
'-v',
'--verbose',
action='store_true',
default=False,
help=('Enables verbose output with -s\n'),
)
parser.add_argument(
'-c',
'--check',
action='store_true',
default=False,
help=('Performs the new server checks instead of tweaks\n'),
)
parser.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help=('Does both server tweaks\n'),
)
parser.add_argument(
'-d',
'--dns-cluster',
action='store_true',
default=False,
help=('Checks dns clustering\n'),
)
self.args = parser.parse_args()
def run_cmd(self, cmd, shell=False):
'''
runs subprocess, builds log var, returns stdout
'''
if shell:
input_str = cmd
else:
input_str = shlex.join(cmd)
self.log(f'EXEC: {input_str}\n')
try:
ret = sp.run(
cmd, stdout=sp.PIPE, text=True, shell=shell, check=False
)
except Exception as e:
error_str = f'Error on cmd:\n{input_str}\nOutput:\n{e}\n'
self.log(error_str)
self.log_str(error_str)
result = ret.stdout
self.log(f'OUT: {result}\n', end=True)
return result
def mail_str(self):
'''
mailing function for str
'''
from_addr = f'root@{self.host_name}'
to_addr = 'str@imhadmin.net'
if self.is_vps:
server_string = 'VPS'
else:
server_string = 'Dedicated Server'
message = (
'From: {0}\n'
'To: {1}\n'
'Subject: New {2} with errors: {3}\n'
'Errors encountered during first_setup.py:\n'
'Host: {3}\n'
'{4}\n\n'.format(
from_addr, to_addr, server_string, self.host_name, self.str_msg
)
)
with smtplib.SMTP_SSL('localhost', 465) as server:
# server.set_debuglevel(True)
server.sendmail(from_addr, to_addr, message.encode('utf-8'))
def final_output(self):
'''
finishes program and handles final output (str/stdout)
'''
if self.str_msg and self.args.send_str:
self.log('sending str\n')
self.mail_str()
if not self.str_msg and self.args.send_str:
sys.exit(0)
print(self.str_msg or 'first_setup complete.')
sys.exit(0)
def log(self, line, end=False):
'''
logging function
'''
if not line:
line = '\n'
if not self.args.send_str or self.args.verbose:
print(line, end='')
self.run_log += line
if end:
bar = '-' * 10 + '\n' # pylint: disable=disallowed-name
self.run_log += bar
if not self.args.send_str or self.args.verbose:
print(bar, end='')
def log_str(self, msg):
self.str_msg += '-' * 10 + '\n'
self.str_msg += msg
def refresh_services(self):
'''
runs after user provisioning
'''
self.run_cmd(('package-cleanup', '-y', '--cleandupes'))
# make sure rsyslog is working
os.remove('/var/lib/rsyslog/imjournal.state')
self.run_cmd(('systemctl', 'restart', 'rsyslog'))
# fix quotas, takes less than a second usually on new cts
self.run_cmd(('/scripts/fixquotas',))
# ensures systemd-logind not causing ssh slowness
self.run_cmd(('systemctl', 'restart', 'systemd-logind'))
def check_clustering(self):
output = self.run_cmd(('du', '-s', '/var/cpanel/cluster'))
if output.startswith('4\t'):
self.log_str(
'Error: DNS clustering is not set up, '
'/var/cpanel/cluster is empty\n'
)
def check_things(self):
'''
checks a few things so the customer wont have to contact us for them
makes an str if issues are found and the -s flag is used (vps auto prov)
otherwise the errors go to stdout (dedi manual prov)
intended to run after user provisioning.
'''
# checks for A record for hostname
output = self.run_cmd(('dig', '+short', '@ns', self.host_name))
if not output:
self.log_str('Error: A record for hostname not found\n')
# checks if dns clustering is unconfigured
# in 15 minutes
delay = '15'
if self.args.dns_cluster:
flags = '-ds'
else:
flags = '-d'
p_cmd = f'/opt/dedrads/first_setup.py {flags}'
p_cmd = p_cmd.encode('utf-8')
with sp.Popen(
('at', 'now', '+', delay, 'minutes'),
stdin=sp.PIPE,
stdout=sp.DEVNULL,
stderr=sp.PIPE,
# encoding='utf-8',
) as proc:
_, stderr = proc.communicate(p_cmd)
if proc.poll():
self.log_str(
'Failed to schedule DNS cluster check with atd. '
f'Error:{stderr}'
)
tweak_cmd = (
'whmapi1',
'set_tweaksetting',
'key=mycnf_auto_adjust_maxallowedpacket',
'value=0',
)
ssl_cmd = (
'/usr/local/cpanel/bin/checkallsslcerts',
'--allow-retry',
'--verbose',
)
# check cpanel license
output = self.run_cmd(('/usr/local/cpanel/cpkeyclt',))
if 'Update succeeded' not in output:
self.log_str(
'Error: New server with invalid '
'cPanel license.\nstdout:\n{}'
'Run these commands after fixing cPanel the license:\n'
'{}\n{}\n'.format(
output,
' '.join(tweak_cmd),
' '.join(ssl_cmd),
)
)
else:
# these require a valid cPanel license to run
self.run_cmd(ssl_cmd)
self.run_cmd(tweak_cmd)
# restarts cpanel to get new service ssl if they were bad and
# are now signed
self.run_cmd('service cpanel restart &>/dev/null', shell=True)
def tune_system(self):
'''
These operations are kinda slow (30 seonds or so)
So they are best to be ran before the customer pays
They also do not require a valid cpanel license
'''
if os.path.exists('/usr/sbin/yum-complete-transaction'):
self.run_cmd(('/usr/sbin/yum-complete-transaction', '-y'))
if self.is_vps:
# Perl dependancies for IMH scripts
self.run_cmd(
(
'yum',
'-y',
'install',
'perl-File-Slurp',
'perl-YAML',
'perl-YAML-Syck',
'perl-Template-Toolkit',
)
)
self.run_cmd(
(
'yes y | cpan -i Switch '
'LWP::Protocol::https '
'IO::Scalar '
'Date::Parse '
'Text::Template '
'CDB_File &>/dev/null'
),
shell=True,
)
else:
# for dedis
self.run_cmd(('cpan', '-i', 'JSON::XS'))
self.run_cmd(('kcarectl', '-u'))
def set_swappiness(self):
'''Sets swappiness to 0 and clears swap if dedicated server'''
if self.is_vps:
return
self.run_cmd(['sysctl', '-w', 'vm.swappiness=0'])
self.log('Swap is now clearing\n')
self.run_cmd(['swapoff', '-a'])
self.run_cmd(['swapon', '-a'])
def tune_ea4(self):
'''
Installs missing Apache modules
'''
if self.args.no_ea4:
return
ea_file = '/etc/cpanel/ea4/profiles/custom/imh.json'
if not os.path.isfile(ea_file):
self.log(
(
'missing ea4 profile, skipping '
'apache module install. path:\n{}\n'.format(ea_file)
),
end=True,
)
ea4_tup = (
'ea-apache24-mod_headers',
'ea-apache24-mod_env',
'ea-apache24-mod_suexec',
'ea-apache24-mod_version',
'ea-apache24-mod_proxy_fcgi',
'ea-apache24-mod_remoteip',
)
after = '"ea-apache24",'
for value in ea4_tup:
search = fr'''\s*("|')?{value}("|')?,'''
value = f''' "{value}",'''
self.conf_set(search, value, ea_file, after=after)
self.run_cmd(
('/usr/local/bin/ea_install_profile', '--install', ea_file)
)
def tune_apache_php(self):
'''Does math to figure out Apache settings'''
min_spare = 5
max_spare = 20
server_limit = int(math.floor(self.mem_total / 46000 / 4))
if server_limit > 150:
server_limit = 150
elif server_limit < 15:
server_limit = 10
# https://httpd.apache.org/docs/2.4/mod/worker.html
# ServerLimit is a hard limit on the number of active child processes
# and must be greater than or equal to the MaxRequestWorkers directive
# divided by the ThreadsPerChild directive."
# server_limit >= max_req / 25
# max_req <= server_limit * 25
max_workers = server_limit * 25
# tested only on the new file from cpanel 82+, might work on the old
# local file though
conf_file = '/etc/cpanel/ea4/ea4.conf'
self.conf_set(
r'(^\s*"minspareservers"\s*:\s*"?)\d+("?)[^$]',
fr'\g<1>{min_spare}\g<2>,',
conf_file,
fallback=False,
)
self.conf_set(
r'(^\s*"maxspareservers"\s*:\s*"?)\d+("?)[^$]',
fr'\g<1>{max_spare}\g<2>,',
conf_file,
fallback=False,
)
self.conf_set(
r'(^\s*"serverlimit"\s*:\s*"?)\d+("?)[^$]',
fr'\g<1>{server_limit}\g<2>,',
conf_file,
fallback=False,
)
self.conf_set(
r'(^\s*"maxclients"\s*:\s*"?)\d+("?)[^$]',
fr'\g<1>{max_workers}\g<2>,',
conf_file,
fallback=False,
)
if self.is_vps:
self.log('Setting FPM default pool settings\n')
# Removed max_children settings due to potential resource abuse.
# Only raises idle timeout and max requests
# Current default max req is 20, timeout is 10
# Leaving max children as default
fpm_config = 'pm_process_idle_timeout : 30\npm_max_requests : 128\n'
try:
os.makedirs('/var/cpanel/ApachePHPFPM')
except (OSError, FileExistsError) as e:
self.log(f"{e}\n")
self.log('Setting Apache global config\n')
path = '/var/cpanel/ApachePHPFPM/system_pool_defaults.yaml'
with open(path, 'w', encoding='utf-8') as fh:
fh.write(fpm_config)
else:
self.log('Skipping FPM default pool settings on Dedicated\n')
self.run_cmd(['/scripts/rebuildhttpdconf'])
self.run_cmd(['/scripts/php_fpm_config', '--rebuild'])
self.run_cmd(['/scripts/restartsrv_apache_php_fpm'])
self.run_cmd('/scripts/restartsrv_apache &>/dev/null', shell=True)
def tune_mysql(self):
'''
sets innodb buffer pool size and instances
maxes out at 4GB of innodb buffer pool
every 1GB gets another pool instance
'''
mem_gate = 0
pool_count = 1
pool_size = 2
mem_tiers = (
(1000000, 1, 256),
(3000000, 1, 512),
(7000000, 1, 1024),
(14000000, 2, 2048),
(30000000, 4, 4096),
)
buffer_pool = 256
pool_instances = 1
for tier in mem_tiers:
if self.mem_total > tier[mem_gate]:
buffer_pool = tier[pool_size]
pool_instances = tier[pool_count]
else:
break
buffer_pool = f'innodb_buffer_pool_size={buffer_pool}M\n'
pool_instances = f'innodb_buffer_pool_instances={pool_instances}\n'
conf_file = '/etc/my.cnf'
self.conf_set(
r'^\s*innodb_buffer_pool_size\s*=[^$]*', buffer_pool, conf_file
)
self.conf_set(
r'^\s*innodb_buffer_pool_instances\s*=[^$]*',
pool_instances,
conf_file,
)
self.run_cmd(['service', 'mysql', 'restart'])
def get_mem_total(self):
'''
Gets total memory
switched to free because /proc/meminfo may not be kb in the future
'''
cmd = '''free |sed -e 1d -e 3d | awk '{print $2}' '''
mem_total = self.run_cmd(cmd, shell=True)
mem_total = int(mem_total)
if not mem_total:
self.log('Failed to check available memory\n', end=True)
sys.exit(1)
self.mem_total = mem_total
def conf_set(
self, _search, _replace, _file, after='', multiple=False, fallback=True
):
'''
configuration management function
'''
after_found = 0
replace_count = 0
base_fn = os.path.basename(_file)
tmp_fn = f'/tmp/{base_fn}.tmp'
with open(tmp_fn, 'w', encoding='utf-8') as tmp_fh:
with open(_file, encoding='utf-8') as og_fh:
og_fh = og_fh.read()
# checks if already set, making it re-runnable
if _replace in og_fh:
self.log(
f'Setting already set in file {_file}:\n{_replace}\n',
end=True,
)
return
og_fh = og_fh.splitlines(True)
for og_line in og_fh:
# detects the after line
if after and not after_found and re.search(after, og_line):
after_found = 1
self.log(f'after line found: {after}\n')
# main conditional gate for doing the replace
if (
(
(not after and _search != 'APPEND')
or (after and after_found)
)
and (multiple or not replace_count)
and (re.search(_search, og_line) or _search == 'APPEND')
):
if _search == 'APPEND':
self.log(
f'appending {_replace} after {after}\n',
end=True,
)
new_line = f'{og_line}{_replace}\n'
tmp_fh.write(new_line)
replace_count += 1
continue
self.log(
'doing traditional search/replace:\n'
f'{_search} -> {_replace}\n',
end=True,
)
new_line = re.sub(_search, _replace, og_line)
tmp_fh.write(new_line)
replace_count += 1
continue
tmp_fh.write(og_line)
if not replace_count and fallback:
if not after:
self.log(
'appending to end of file as fallback:\n{}\n'.format(
_replace
),
end=True,
)
replace_count += 1
tmp_fh.write(f'{_replace}\n')
else:
self.log('doing recurisve conf_set for after fallback\n')
os.remove(tmp_fn)
self.conf_set(
'APPEND', _replace, _file, after=after, fallback=False
)
return
if not replace_count:
self.log(
(
'Warning: Failed to replace (fallback False):\n'
'search:{}\nreplace:{}\n'
'file:{}\nafter:{}\n'.format(
_search, _replace, _file, after
)
),
end=True,
)
os.remove(tmp_fn)
return
shutil.move(tmp_fn, _file)
def is_a_vps(self):
if os.path.exists('/proc/vz/veinfo'):
self.is_vps = True
else:
self.is_vps = False
def main():
"""if -c is used, the first section runs this is intended to be ran after
the customers user is added without -c is meant to be ran before the
customer is provisioned (clonevps /ready dedi)"""
setup = FirstSetup()
if setup.args.all:
# does all the config changes
setup.set_swappiness()
setup.tune_system()
setup.tune_ea4()
setup.refresh_services()
setup.check_things()
setup.tune_apache_php()
setup.tune_mysql()
elif setup.args.check:
# requires cpanel license / vps package must be set / is fast
setup.refresh_services()
setup.check_things()
setup.tune_apache_php()
setup.tune_mysql()
elif setup.args.dns_cluster:
setup.check_clustering()
else:
# stuff to be performed on clones/ready dedis before purchase
setup.set_swappiness()
setup.tune_system()
setup.tune_ea4()
setup.final_output()
if __name__ == '__main__':
main()