File: //usr/syno/sbin/synologconfgen
#!/usr/bin/env python3
import sys
import os
import re
import errno
import json
import shutil
import configparser
import tempfile
from importlib.machinery import SourceFileLoader
import fcntl
import getopt
import subprocess
import time
DEV_MODE = False
CONFIG_LOCK_TIMEOUT = 60
CONFIG_LOCK_INTERVAL = 0.5
class SynoException(Exception):
pass
class Path(object):
@staticmethod
def mkdir_p(dir_path, mode):
try:
os.makedirs(dir_path, mode)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(dir_path):
pass
else:
raise
@staticmethod
def rm_f(file_path):
try:
os.unlink(file_path)
except OSError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
@staticmethod
def rm_rf(dir_path):
try:
shutil.rmtree(dir_path)
except OSError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
@staticmethod
def ls(dir_path):
try:
filenames = os.listdir(dir_path)
except OSError:
filenames = []
return filenames
@staticmethod
def ls_file_paths(dir_rel_path, file_suffix=None):
dir_paths = [
os.path.join(Path.DSM_PREFIX, dir_rel_path),
os.path.join(Path.PKG_PREFIX, dir_rel_path),
]
file_paths = []
for dir_path in dir_paths:
for filename in Path.ls(dir_path):
if file_suffix and not filename.endswith(file_suffix):
continue
file_paths.append(os.path.join(dir_path, filename))
return file_paths
DSM_PREFIX = '/usr/syno'
PKG_PREFIX = '/usr/local'
class SubGenerator(object):
LOGCENTER_DIR = '/var/packages/LogCenter/target/synologconf/subgens/'
def __init__(self, file_path):
module_name = self._extract_module_name(file_path)
module = self._load_module(module_name, file_path)
self._set_module_functions(module)
self.name = module_name
self.path = file_path
@staticmethod
def load_all():
sub_gens = []
logcenter_names, file_paths = SubGenerator._find_conf_in_logcenter()
for file_path in Path.ls_file_paths(SubGenerator.DIR_REL_PATH, '.py'):
# Prefer to synologconf subgens in LogCenter package
if os.path.basename(file_path) in logcenter_names:
continue
file_paths.append(file_path)
for file_path in file_paths:
try:
sub_gen = SubGenerator(file_path)
except SynoException as e:
print('ERROR: load sub-generator "{}" failed'.format(file_path))
print(' {}'.format(e))
continue
sub_gens.append(sub_gen)
print('INFO: loaded sub-generator "{}" from "{}"'.format(sub_gen.name, sub_gen.path))
return sub_gens
@staticmethod
def _find_conf_in_logcenter():
dir_path = SubGenerator.LOGCENTER_DIR
file_prefix = 'logcenter'
file_suffix = '.py'
file_paths = []
file_names = []
for filename in Path.ls(dir_path):
if not filename.startswith(file_prefix) or not filename.endswith(file_suffix):
continue
file_names.append(filename)
file_paths.append(os.path.join(dir_path, filename))
return file_names, file_paths
def _extract_module_name(self, file_path):
filename = os.path.basename(file_path)
match = re.match(r'(.+)\.py$', filename)
if not match:
raise SynoException('invalid file path')
return match.group(1)
def _load_module(self, module_name, file_path):
try:
return SourceFileLoader(module_name, file_path).load_module()
except Exception as e:
raise SynoException(e)
def _def_generate_syslog_ng_conf(self, synolog_conf):
return False
def _set_module_functions(self, module):
try:
self.validate_synolog_conf = module.validate_synolog_conf
try:
self.generate_syslog_ng_conf = module.generate_syslog_ng_conf
except AttributeError as e:
raise SynoException('missing required function, {}'.format(e))
except AttributeError as e:
self.validate_synolog_conf = self._def_generate_syslog_ng_conf
try:
self.generate_common_syslog_ng_conf = module.generate_common_syslog_ng_conf
except AttributeError as e:
self.generate_common_syslog_ng_conf = self._def_generate_syslog_ng_conf
DIR_REL_PATH = 'lib/synosyslog/subgens'
class SynoLogCommonConf(object):
def __init__(self, app_id, path):
self.type = SyslogNgConf.CONF_TYPE_COMMON
self.app_id = app_id
self.path = path
def generate(self, sub_gen):
syslog_ng_conf = SyslogNgConf(self)
sub_gen_result = False
try:
sub_gen_result = sub_gen.generate_common_syslog_ng_conf(self)
except Exception as e:
print('ERROR: failed to run "generate_common_syslog_ng_conf" in "{}" on app "{}"'.format(self.path, self.app_id))
print(' {}'.format(e))
if DEV_MODE:
raise
if not sub_gen_result:
return False
syslog_ng_conf.add(sub_gen_result, sub_gen.name)
return syslog_ng_conf
class SynoLogConf(object):
def __init__(self, conf_path):
self.type = SyslogNgConf.CONF_TYPE_APP
self.app_id = self._extract_app_id(conf_path)
self.conf = self._load_conf(conf_path)
self.mappings = self._load_mappings(conf_path)
self.path = conf_path
def is_key_enabled(self, key):
if key not in self.conf:
return False
return bool(self.conf[key])
def validate(self, sub_gens):
valid_sub_gens = []
for sub_gen in sub_gens:
try:
success = sub_gen.validate_synolog_conf(self)
except Exception as e:
print('ERROR: failed to run "validate_synolog_conf" in "{}" on app "{}"'.format(sub_gen.name, self.app_id))
print(' {}'.format(e))
if DEV_MODE:
raise
continue
if success:
valid_sub_gens.append(sub_gen)
else:
print('WARN: synolog conf for app "{}" did not pass the validation in "{}"'.format(self.app_id, sub_gen.name))
return valid_sub_gens
def generate(self, sub_gens):
syslog_ng_conf = SyslogNgConf(self)
for sub_gen in sub_gens:
try:
sub_gen_result = sub_gen.generate_syslog_ng_conf(self)
except Exception as e:
print('ERROR: failed to run "generate_syslog_ng_conf" in "{}" on app "{}"'.format(sub_gen.name, self.app_id))
print(' {}'.format(e))
if DEV_MODE:
raise
continue
syslog_ng_conf.add(sub_gen_result, sub_gen.name)
return syslog_ng_conf
@staticmethod
def load_all():
synolog_confs = []
for file_path in Path.ls_file_paths(SynoLogConf.DIR_REL_PATH):
try:
synolog_conf = SynoLogConf(file_path)
except SynoException as e:
print('ERROR: load synolog config "{}" failed'.format(file_path))
print(' {}'.format(e))
continue
synolog_confs.append(synolog_conf)
print('INFO: loaded synolog config "{}" from "{}"'.format(synolog_conf.app_id, synolog_conf.path))
return synolog_confs
def _extract_app_id(self, conf_path):
return os.path.basename(conf_path)
def _load_conf(self, conf_path):
try:
with open(conf_path) as f:
return json.load(f)
except IOError:
raise SynoException('open synolog config failed: {}'.format(conf_path))
except ValueError:
raise SynoException('parse json file failed: {}'.format(conf_path))
def _load_mappings(self, conf_path):
base_dir_path = os.path.dirname(os.path.dirname(conf_path))
event_path = os.path.join(base_dir_path, 'events', self.app_id)
parser = configparser.ConfigParser()
try:
with open(event_path) as f:
parser.read_file(f)
except IOError:
raise SynoException('open synolog event file failed: {}'.format(event_path))
except configparser.Error:
raise SynoException('parse synolog event file failed: {}'.format(event_path))
mappings = {}
for section in parser.sections():
mappings[section] = {}
for option in parser.options(section):
if option[0] != '@':
continue
try:
arg_seq = int(option[1:])
except ValueError:
continue
mappings[section][arg_seq] = parser.get(section, option)
return mappings
DIR_REL_PATH = 'share/synolog/configs'
class SyslogNgConf(object):
def __init__(self, synolog_conf):
self.synolog_conf = synolog_conf
self.path = self._get_dump_path()
self.confs = []
def add(self, sub_gen_result, sub_gen_name):
if not sub_gen_result:
return
try:
sub_gen_confs = self._make_sub_gen_confs(sub_gen_result)
except SynoException as e:
print('ERROR: generated syslog-ng conf from "{}" on app "{}" is malformed, skipped'.format(sub_gen_name, self.synolog_conf.app_id))
print(' {}'.format(e))
return
try:
for sub_gen_conf in sub_gen_confs:
self._validate_sub_gen_conf(sub_gen_conf)
except SynoException as e:
print('ERROR: generated syslog-ng conf from "{}" on app "{}" is invalid, skipped'.format(sub_gen_name, self.synolog_conf.app_id))
print(' {}'.format(e))
return
if self.synolog_conf.type == SyslogNgConf.CONF_TYPE_COMMON:
self.confs = sub_gen_confs
return
for idx, sub_gen_conf in enumerate(sub_gen_confs):
source_conf, other_conf = self._split_source_from_conf(sub_gen_conf)
if not source_conf:
self.confs.append(self._gen_synosyslog_base_conf(sub_gen_name, idx) + other_conf)
else:
self.confs.append(source_conf +
self._gen_parser_base_conf(sub_gen_name, idx) +
self._gen_args_base_conf(sub_gen_name, idx) +
other_conf)
def _split_source_from_conf(self, sub_gen_conf):
source_conf = []
other_conf = []
for entry in sub_gen_conf:
if entry['type'] == 'source':
source_conf.append(entry)
else:
other_conf.append(entry)
return source_conf, other_conf
def dump(self):
contents = []
contents.append(SyslogNgConf.GENERATED_WARNING)
for conf in self.confs:
contents.extend(self._render_components(conf))
if self.synolog_conf.type == SyslogNgConf.CONF_TYPE_APP:
for conf in self.confs:
contents.append(self._render_flow(conf))
contents = self._remove_duplicated(contents)
try:
with open(self.path, 'w') as f:
f.write('\n\n'.join(contents) + '\n')
os.chmod(self.path, 0o644)
except IOError:
Path.rm_f(self.path)
raise SynoException('write syslog-ng config failed: {}'.format(self.path))
@staticmethod
def make_temp_dir():
SyslogNgConf.CONF_DIR_PATH_TMP = SyslogNgConf.CONF_DIR_PATH + ".gen.tmp"
SyslogNgConf.CONF_DIR_PATH_TMP_SYSTEM = SyslogNgConf.CONF_DIR_PATH_TMP + '/' + SyslogNgConf.CONF_TYPE_SYSTEM
SyslogNgConf.CONF_DIR_PATH_TMP_COMMON = SyslogNgConf.CONF_DIR_PATH_TMP + '/' + SyslogNgConf.CONF_TYPE_COMMON
SyslogNgConf.CONF_DIR_PATH_TMP_APP = SyslogNgConf.CONF_DIR_PATH_TMP + '/' + SyslogNgConf.CONF_TYPE_APP
Path.rm_rf(SyslogNgConf.CONF_DIR_PATH_TMP)
Path.mkdir_p(SyslogNgConf.CONF_DIR_PATH_TMP_SYSTEM, 0o755)
Path.mkdir_p(SyslogNgConf.CONF_DIR_PATH_TMP_COMMON, 0o755)
Path.mkdir_p(SyslogNgConf.CONF_DIR_PATH_TMP_APP, 0o755)
os.chmod(SyslogNgConf.CONF_DIR_PATH_TMP, 0o755)
os.chmod(SyslogNgConf.CONF_DIR_PATH_TMP_SYSTEM, 0o755)
os.chmod(SyslogNgConf.CONF_DIR_PATH_TMP_COMMON, 0o755)
os.chmod(SyslogNgConf.CONF_DIR_PATH_TMP_APP, 0o755)
@staticmethod
def move_temp_to_gen():
try:
shutil.move(SyslogNgConf.CONF_DIR_PATH, SyslogNgConf.CONF_DIR_PATH + ".move.tmp")
except IOError as e:
if e.errno == errno.ENOENT:
pass
else:
Path.rm_rf(SyslogNgConf.CONF_DIR_PATH + '/' + SyslogNgConf.CONF_TYPE_APP)
Path.rm_rf(SyslogNgConf.CONF_DIR_PATH)
shutil.move(SyslogNgConf.CONF_DIR_PATH_TMP, SyslogNgConf.CONF_DIR_PATH)
Path.rm_rf(SyslogNgConf.CONF_DIR_PATH + ".move.tmp")
Path.rm_rf(SyslogNgConf.CONF_DIR_PATH_TMP)
def _get_dump_path(self):
return os.path.join(SyslogNgConf.CONF_DIR_PATH_TMP, self.synolog_conf.type, self.synolog_conf.app_id)
def _gen_synosyslog_base_conf(self, sub_gen_name, idx):
return [
{
'type': 'source',
'name': 's_syno_synosyslog',
'content': None
},
{
'type': 'filter',
'name': 'f_syno_{}'.format(self.synolog_conf.app_id),
'content': [
SyslogNgConf.FILTER_FMT.format(self.synolog_conf.app_id)
]
}] + self._gen_parser_base_conf(sub_gen_name, idx) + [{
'type': 'rewrite',
'name': 'r_syno_username',
'content': None
}] + self._gen_args_base_conf(sub_gen_name, idx)
def _gen_parser_base_conf(self, sub_gen_name, idx):
return [
{
'type': 'parser',
'name': 'p_syno_sdata_{}_{}_{}'.format(self.synolog_conf.app_id, sub_gen_name, idx),
'content': [
SyslogNgConf.SYNOSDATA_FMT.format(sd_tag=SyslogNgConf.SYNO_SD_TAG)
]
},
]
def _gen_args_base_conf(self, sub_gen_name, idx):
return [{
'type': 'rewrite',
'name': 'r_syno_args_{}'.format(self.synolog_conf.app_id),
'content': self._gen_args_rewrite_content()
},
]
def _gen_args_rewrite_content(self):
content = []
for event_id, mapping in self.synolog_conf.mappings.items():
for arg_seq, arg_name in mapping.items():
content.append(SyslogNgConf.ARGS_REWRITE_FMT.format(**{
'event_id': event_id,
'arg_seq': arg_seq,
'arg_name': arg_name,
}))
return content
def _make_sub_gen_confs(self, sub_gen_result):
if type(sub_gen_result) is not list:
raise SynoException('should be a list')
if len(sub_gen_result) == 0:
raise SynoException('should not be empty list')
first = sub_gen_result[0]
if type(first) is list:
return sub_gen_result
elif type(first) is dict:
return [sub_gen_result]
else:
raise SynoException('unknown list element')
def _validate_sub_gen_conf(self, sub_gen_conf):
if type(sub_gen_conf) is not list:
raise SynoException('should be a list')
for entry in sub_gen_conf:
if type(entry) is not dict:
raise SynoException('list element should be a dict')
for field in ['type', 'name', 'content']:
if field not in entry:
raise SynoException('each entry should contain the field "{}"'.format(field))
if 'destination' not in [entry['type'] for entry in sub_gen_conf]:
raise SynoException('should be at least one destination')
def _render_components(self, conf):
contents = []
for entry in conf:
if entry['content'] is None:
continue
component = self._render_component(entry)
contents.append(component)
return contents
def _render_component(self, entry):
lines = [SyslogNgConf.COMPONENT_ITEM_FMT.format(line) for line in entry['content']]
return SyslogNgConf.COMPONENT_FMT.format(lines='\n'.join(lines), **entry)
def _render_flow(self, conf):
lines = [SyslogNgConf.FLOW_ITEM_FMT.format(**entry) for entry in conf]
return SyslogNgConf.FLOW_FMT.format(lines='\n'.join(lines))
def _remove_duplicated(self, ori_items):
new_items = []
for item in ori_items:
if item in new_items:
continue
new_items.append(item)
return new_items
CONF_DIR_PATH = '/etc/syslog-ng/syno.d/synosyslog.conf.gen'
CONF_DIR_PATH_TMP = ''
CONF_DIR_PATH_TMP_SYSTEM = ''
CONF_DIR_PATH_TMP_COMMON = ''
CONF_DIR_PATH_TMP_APP = ''
CONF_TYPE_SYSTEM = 'system'
CONF_TYPE_COMMON = 'common'
CONF_TYPE_APP = 'app'
SYNO_SD_TAG = 'synolog@6574'
GENERATED_WARNING = '''# Generated by synologconfgen
# DO NOT EDIT THIS FILE BY HAND - YOUR CHANGES WILL BE OVERWRITTEN'''
COMPONENT_FMT = '''{type} {name} {{
{lines}
}};'''
FLOW_FMT = '''log {{
{lines}
}};'''
COMPONENT_ITEM_FMT = ' {}'
FLOW_ITEM_FMT = ' {type}({name});'
FILTER_FMT = 'program("^{}$");'
SYNOSDATA_FMT = 'json-parser(template("$(format-json --key .SDATA.{sd_tag}.* --replace-prefix .SDATA.{sd_tag}.=)") prefix("SYNOSDATA."));'
ARGS_REWRITE_FMT = 'set("${{SYNOSDATA.arg_{arg_seq}}}", value("ARGS.{arg_name}") condition(match("0x{event_id}" value("SYNOSDATA.event_id"))));'
class FileLock():
def __init__(self):
self._lock_fd = -1
def __del__(self):
self.release()
def acquire(self, file_path, timeout):
count = int(timeout / CONFIG_LOCK_INTERVAL)
count = count if count > 1 else 1
for _ in range(count):
self._acquire(file_path)
if self._lock_fd != -1:
break
time.sleep(CONFIG_LOCK_INTERVAL)
def _acquire(self, file_path):
fd = os.open(file_path, os.O_RDWR | os.O_CREAT, 0o444)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except (OSError, IOError):
os.close(fd)
else:
self._lock_fd = fd
def release(self):
if self._lock_fd != -1:
try:
fcntl.flock(self._lock_fd, fcntl.LOCK_UN)
except OSError:
pass
os.close(self._lock_fd)
self._lock_fd = -1
# Don't backup if support_xa is defined to avoid disk failure
def need_backup():
with open('/etc.defaults/synoinfo.conf', 'r') as f:
data = f.read()
if 'support_xa="yes"' not in data:
return True
return False
def gen_backup_configs():
if need_backup():
path = SyslogNgConf.CONF_DIR_PATH_TMP_SYSTEM + '/diskutil_system'
with open(path, 'w') as f:
config = "destination d_diskutil_system { d_synosys_block(db_path('/var/lib/diskutil/system')); };\nlog { source(s_syno_syslog); filter(f_synosys); parser(p_msg_to_user_event); destination(d_diskutil_system); };"
f.write(config)
os.chmod(path, 0o644)
def run():
file_lock = FileLock()
file_lock.acquire("/var/run/synologconfgen.lock", CONFIG_LOCK_TIMEOUT)
SyslogNgConf.make_temp_dir()
sub_gens = SubGenerator.load_all()
for sub_gen in sub_gens:
synolog_conf = SynoLogCommonConf(sub_gen.name, sub_gen.path)
syslog_ng_conf = synolog_conf.generate(sub_gen)
if not syslog_ng_conf:
continue
try:
syslog_ng_conf.dump()
print('INFO: generated syslog-ng config for app "{}" on "{}"'.format(synolog_conf.app_id, syslog_ng_conf.path))
except SynoException as e:
print('WARN: skipped generating syslog-ng config for app "{}"'.format(synolog_conf.app_id))
print(' {}'.format(e))
synolog_confs = SynoLogConf.load_all()
for synolog_conf in synolog_confs:
valid_sub_gens = synolog_conf.validate(sub_gens)
syslog_ng_conf = synolog_conf.generate(valid_sub_gens)
try:
syslog_ng_conf.dump()
print('INFO: generated syslog-ng config for app "{}" on "{}"'.format(synolog_conf.app_id, syslog_ng_conf.path))
except SynoException as e:
print('WARN: skipped generating syslog-ng config for app "{}"'.format(synolog_conf.app_id))
print(' {}'.format(e))
gen_backup_configs()
SyslogNgConf.move_temp_to_gen()
file_lock.release()
print('INFO: syslog-ng config generation for synolog is done')
def main():
global DEV_MODE
opts, _ = getopt.getopt(sys.argv[1:], '-d', ['dev'])
for opt_name, opt_value in opts:
if opt_name in ('-d', '--dev'):
DEV_MODE = True
run()
if __name__ == '__main__':
main()