#!/bin/bash
system_dump_script=syno_system_dump

#log files, remember to delete at "disable"
log_file=/var/log/syno_sys_status.log
skip_log_file=/var/log/syno_sys_status_skip.log
dump_tmp_log_file=/var/log/dump_tmp_files.log
#log files, remember to delete at "disable"

dump_running=/var/run/lock/syno_system_dump.lock
tehuti_low_mem_counter=/proc/driver/syno_tehuti_low_mem_counter

dump_D_process_call_stack()
{
	echo "[date] $(date)"
	echo [D state process call stack]
	PIDS=$(ps -e -T -o stat,tid | grep ^D | awk '{print $2}')
	for i in ${PIDS[@]}; do
		echo "==========="
		cmd=$(tr -d '\0' <"/proc/${i}/cmdline")
		echo "PID: ${i}  cmdline:${cmd}"
		cat "/proc/${i}/stack"
	done
	echo ""
}

get_gateway()
{
	#    route -n |
	while IFS=$'\n' read line
	do
		echo "$line" | awk '{print $4}' |grep G > /dev/null
		if [ 0 -eq $? ]; then
			gateway=$(echo "$line" | awk '{print $2}')
			break
		fi
	done < <(route -n)
}

dump_io_rank()
{
	echo [read]
	{
		IFS= read -r header
		sort -b -k4 -r | head -n 20
	} < <(pidstat -d)

	echo ""
	echo [write]
	{
		IFS= read -r header
		sort -b -k5 -r | head -n 20
	} < <(pidstat -d)
}

dump_nfs_io()
{
	local cmd_lat_file="/proc/net/rpc/nfsd_lat"
	local vfs_lat_file="/proc/fs/nfsd/syno_io_stat"
	local rw_lat_hist_file="/proc/fs/nfsd/syno_latency_histogram"

	if [[ -f ${cmd_lat_file} ]]; then
		echo "[nfs command latency]"
		cat ${cmd_lat_file}
		echo ""
	fi

	if [[ -f ${vfs_lat_file} ]]; then
		echo "[nfs vfs latency]"
		cat ${vfs_lat_file}
		echo ""
	fi

	if [[ -f ${rw_lat_hist_file} ]]; then
		echo "[nfs read/write latency histogram]"
		cat ${rw_lat_hist_file}
		echo ""
	fi
}

dump_ethtool_statistics()
{
	local dev_name=""
	for name in /sys/class/net/*; do
		dev_name=$(basename "$name")
		# prevent to dump VLAN ex:eth0.100
		if [[ "$dev_name" == eth* ]] && [[ ! "$dev_name" == *.* ]]; then
			echo [$dev_name]
			ethtool -S $dev_name
		fi
	done
}

run_dump()
{
	echo "[date] $(date)"
	echo [ps result]
	ps axwf
	echo ""

	echo "[systemd job]"
	/bin/systemctl list-jobs --no-pager
	echo ""

	echo "[systemd failed unit]"
	/bin/systemctl --failed --no-pager --no-legend
	echo ""

	echo [vmstat result]
	cat /proc/vmstat
	echo ""

	echo [interrupts result]
	cat /proc/interrupts
	echo ""

	echo [slabinfo result]
	cat /proc/slabinfo
	echo ""

	echo [memoryinfo result]
	cat /proc/meminfo
	echo ""

	echo [zoneinfo result]
	cat /proc/zoneinfo
	echo ""

	echo [swap usage rank]
	for file in /proc/*/status ; do awk '/VmSwap|Name/{printf $2 " " $3}END{ print ""}' $file; done | sort -k 2 -n -r | head -n 10
	echo ""

	echo [rss usage rank]
	ps -e -orss,pid=,user=,args=, | sort -b -k1,1n | tail -20 | tac
	echo ""

	echo [cpu usage rank]
	{
		IFS= read -r header
		echo $header
		sort -b -k1,1n | tail -20 | tac
	} < <(ps -e -o%cpu,pid=,user=,args=,)
	echo ""

	if [[ -f "/bin/pidstat" ]]; then
		echo [I/O usage rank]
		dump_io_rank
		echo ""
	fi

	echo [df result]
	df_result=$(df -x fuse.gvfsd-fuse -x cifs -x nfs4 -x nfs)
	echo "$df_result"
	echo ""

	echo [df inode result]
	df -x fuse.gvfsd-fuse -x cifs -x nfs4 -x nfs -i
	echo ""

	echo [top result]
	/usr/bin/top -b -n 2 -w 512
	echo ""

	echo [iostat result]
	/bin/iostat -dxm 1 -t 5
	echo ""

	dump_nfs_io

	echo [route result]
	route -n
	echo ""

	echo [ping gateway]
	get_gateway
	if [ ! -z "$gateway" ]; then
		ping -c 3 "$gateway"
		echo ""
	fi

	echo [ifconfig result]
	ifconfig -a
	echo ""

	echo [ethtool statistics]
	dump_ethtool_statistics
	echo ""

	echo [/tmp big files]
	find /tmp -printf '%s %p\n' | sort -nr | head
	echo ""

	echo [/tmp big documents]
	du -d1 /tmp/ | sort -nr | head
	echo ""

	tmp_percentage=$(echo "$df_result" | grep "% /tmp" | awk '{print $5}' | sed 's/%//')
	if [ "$tmp_percentage" -ge 95 ]; then
		if [ ! -f /var/run/dump_tmp_time ] || find /var/run/dump_tmp_time -mmin +70 | grep -q dump_tmp_time; then
			touch /var/run/dump_tmp_time
			{
				echo "[date] $(date)"
				echo "[find /tmp]"
				find /tmp -printf '%s\t%p\n'
			} >> $dump_tmp_log_file
		fi
	fi

	if [ -e $tehuti_low_mem_counter ]; then
		echo [tehuti packet loss counter]
		cat $tehuti_low_mem_counter
		echo ""
	fi

	echo [HW monitor]
	cat /run/hwmon/*
	echo ""

	echo ""
	echo ""
}

skip()
{
	echo [date] "$(date)"
	echo [ps result]
	ps axwf
	echo
}

add_crontab()
{
	/usr/syno/bin/support_syno_system_dump_schedule --add
}

remove_crontab()
{
	/usr/syno/bin/support_syno_system_dump_schedule --delete
}

if [ "$1" = "enable" ]; then
	add_crontab
elif [ "$1" = "disable" ]; then
	remove_crontab
	/bin/rm -f "$log_file"* "$skip_log_file"* "$dump_tmp_log_file"*
else
	if [ "$1" = "DState" ]; then
		DState="true"
	fi
	(
	if flock -n -x 9
	then
		if [ "$DState" = "true" ]; then
			dump_D_process_call_stack >> $log_file
		else
			run_dump >> $log_file
		fi
	else
		skip >> $skip_log_file
	fi
	) 9>$dump_running
fi