1.1
关于内存
关于CPU
进程PID的CPU高于100%
# top
top - 11:04:04 up 193 days, 7:39, 3 users, load average: 1.37, 0.76, 1.28
Tasks: 236 total, 1 running, 235 sleeping, 0 stopped, 0 zombie
%Cpu(s): 27.0 us, 2.2 sy, 0.0 ni, 70.7 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 16266080 total, 3213824 free, 4364708 used, 8687548 buff/cache
KiB Swap: 8257532 total, 7056140 free, 1201392 used. 10636948 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
29768 root 20 0 6608432 1.6g 31004 S 232.9 10.3 1:07.28 java
# top -Hp 29768
top - 11:04:45 up 193 days, 7:40, 3 users, load average: 1.50, 0.87, 1.29
Threads: 15 total, 1 running, 14 sleeping, 0 stopped, 0 zombie
%Cpu(s): 10.7 us, 2.5 sy, 0.0 ni, 86.9 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 16266080 total, 2910256 free, 4666360 used, 8689464 buff/cache
KiB Swap: 8257532 total, 7056140 free, 1201392 used. 10336304 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
29773 root 20 0 6881048 1.9g 31300 R 99.9 12.2 0:59.26 java
29768 root 20 0 6881048 1.9g 31300 S 0.0 12.2 0:00.01 java
processor : 7
vendor_id : GenuineIntel
cpu family : 6
model : 62
model name : Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz
stepping : 4
microcode : 0x42e
cpu MHz : 2599.999
cache size : 20480 KB
physical id : 14
siblings : 1
core id : 0
cpu cores : 1
apicid : 14
initial apicid : 14
fpu : yes
fpu_exception : yes
cpuid level : 13
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc eagerfpu pni pclmulqdq ssse3 cx16 pcid sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm ssbd ibrs ibpb stibp fsgsbase tsc_adjust smep arat spec_ctrl intel_stibp flush_l1d arch_capabilities
bogomips : 5199.99
clflush size : 64
cache_alignment : 64
address sizes : 43 bits physical, 48 bits virtual
power management:
[root@zz-devbuild01 ~]# cat /proc/cpuinfo
# jstack -l 29768 > ./jstack.log
29768: Unable to open socket file: target process not responding or HotSpot VM not loaded
The -F option can be used when the target process is not responding
# ps -ef | grep 29768
root 29768 29760 99 11:03 pts/1 00:05:39 /usr/local/jdk1.8.0_271/bin/java -classpath /usr/local/apache-maven-3.2.2/boot/plexus-classworlds-2.5.1.jar -Dclassworlds.conf=/usr/local/apache-maven-3.2.2/bin/m2.conf -Dmaven.home=/usr/local/apache-maven-3.2.2 org.codehaus.plexus.classworlds.launcher.Launcher clean package install -Dmaven.test.skip=true -U
// 该启动命令对应如下容器资源56507fb68405
# docker run --cpus=4 -t -d -u 0:0 -v /root/.m2/repository:/root/.m2/repository -v /data/jenkins/data/workspace/tomp-main_4409/tomp-main_4409/settings.xml:/root/.m2/settings.xml -w /data/jenkins/data/workspace/tomp-main_4409 -v /data/jenkins/data/workspace/tomp-main_4409:/data/jenkins/data/workspace/tomp-main_4409:rw,z -v /data/jenkins/data/workspace/tomp-main_4409@tmp:/data/jenkins/data/workspace/tomp-main_4409@tmp:rw,z myharbor.zznode.com/public/maven:3.2.2-jdk-1.8 cat
// 该启动命令对应如下容器资源86afe44edd0c
# docker run --cpus=2 -t -d -u 0:0 -v /root/.m2/repository:/root/.m2/repository -v /data/jenkins/data/workspace/tomp-main_4409/tomp-main_4409/settings.xml:/root/.m2/settings.xml -w /data/jenkins/data/workspace/tomp-main_4409 -v /data/jenkins/data/workspace/tomp-main_4409:/data/jenkins/data/workspace/tomp-main_4409:rw,z -v /data/jenkins/data/workspace/tomp-main_4409@tmp:/data/jenkins/data/workspace/tomp-main_4409@tmp:rw,z myharbor.zznode.com/public/maven:3.2.2-jdk-1.8 cat
# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
86afe44edd0c myharbor.zznode.com/public/maven:3.2.2-jdk-1.8 "cat" 26 minutes ago Up 26 minutes focused_pare
56507fb68405 myharbor.zznode.com/public/maven:3.2.2-jdk-1.8 "cat" 29 minutes ago Up 29 minutes gifted_proskuriakova
# docker stats --no-stream gifted_proskuriakova
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
56507fb68405 gifted_proskuriakova 236.86% 2.177GiB / 15.51GiB 14.04% 1.2kB / 0B 36MB / 65.3MB 17
# docker exec -it 56507fb68405 /bin/sh
/data/jenkins/data/workspace/tomp-main_4409 # java -version
java version "1.8.0_271"
Java(TM) SE Runtime Environment (build 1.8.0_271-b09)
Java HotSpot(TM) 64-Bit Server VM (build 25.271-b09, mixed mode)
/data/jenkins/data/workspace/tomp-main_4409 # cat /etc/os-release
NAME="Alpine Linux"
ID=alpine
VERSION_ID=3.12.3
PRETTY_NAME="Alpine Linux v3.12"
HOME_URL="https://alpinelinux.org/"
BUG_REPORT_URL="https://bugs.alpinelinux.org/"
/data/jenkins/data/workspace/tomp-main_4409 # ps
PID USER TIME COMMAND
1 root 0:00 cat
34 root 0:00 /bin/sh
41 root 8:02 /usr/local/jdk1.8.0_271/bin/java -classpath /usr/local/apache-maven-3.2.2/boot/plexus-classworlds-2.5.1.jar -Dclassworlds.conf=/usr/local/apac
113 root 0:00 /bin/sh
135 root 0:00 ps
/data/jenkins/data/workspace/tomp-main_4409 #
/data/jenkins/data/workspace/tomp-main_4409 # top -Hp 41
top: unrecognized option: p
BusyBox v1.31.1 () multi-call binary.
Usage: top [-bm] [-n COUNT] [-d SECONDS]
Provide a view of process activity in real time.
Read the status of all processes from /proc each SECONDS
and display a screenful of them.
Keys:
N/M/P/T: show CPU usage, sort by pid/mem/cpu/time
S: show memory
R: reverse sort
1: toggle SMP
Q,^C: exit
Options:
-b Batch mode
-n N Exit after N iterations
-d SEC Delay between updates
-m Same as 's' key
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 19:54 min
[INFO] Finished at: 2023-10-24T17:47:20+08:00
[INFO] Final Memory: 506M/2988M
[INFO] ------------------------------------------------------------------------
/data/jenkins/data/workspace/tomp-main_4409/service/tomp-main # mvn clean package install '-Dmaven.test.skip=true' -U
// 如上是涉及到的相关日志
// 问题描述
研发自动打包平台编译阶段jenkins构建服务器某一slave节点在执行编译时，top系统CPU资源占比较高！
// 排查思路
查看占比高的进程-定位到是编译阶段的进程-尝试通过jstack输出堆栈日志-通过top -Hp PID查看占比较高的资源-检查其关联关系；
排查到该命令实际是执行在启动的docker容器中-由于基础镜像无法使用top -Hp PID，在宿主机上执行上述的检查操作；
将模拟数据转移到其他服务器做测试（未测试）【可以升级docker版本重启docker等方式做其他校验测试】这里后端依赖较大较多的本地文件，所以没做相关测试；
通过增加docker 启动命令中对cpu的限制，检查实际的CPU资源消耗；
# docker run --cpus=2 使用该限制时，发现java PID行%CPU值小于等于200%
# docker run --cpus=4 使用该限制时，发现java PID行%CPU值小于等于400%
// 解决方式
因为容器启动没有对资源做限制且编译的项目较大处理的文件较多执行的时间较长，所以CPU持续负载较高；
因此，可以增加容器启动的资源限制；
// 总结，技术自检等
top命令各个参数的含义；
top命令执行结果输出各个键值的含义；
top中CPU相关的占比和所在服务器逻辑CPU之间的关系；
CPU 物理与逻辑核数确认；
docker 资源限制相关参数的使用；
Java JVM堆栈相关信息排查确认；
关于磁盘
软件包管理
系统配置
网络配置