Blog 1307723433353: lipca 2017

## We need to know kernel log
root@ems01:/home/user/TIBCO_HOME# dmesg > dmesg.txt

## We need to what's going on in memory subsystem
root@ems01:/home/user/TIBCO_HOME# vmstat
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
r b swpd free buff cache si so bi bo in cs us sy id wa st
2 0 0 2573920 225408 1240940 0 0 874 374 677 9536 12 3 84 1 0

## We need to what's going on with I/O
root@ems01:/home/user/TIBCO_HOME# iostat
Linux 4.10.0-26-generic (ems01) 07/14/17 _x86_64_ (2 CPU)

avg-cpu: %user %nice %system %iowait %steal %idle
10.65 1.01 3.49 0.72 0.00 84.14

Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn
sda 92.05 1519.83 587.98 1065413 412180
sdb 1.36 184.28 141.54 129180 99224

## We need to know how filesystems are mounted
root@ems01:/home/user/TIBCO_HOME# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=3030152k,nr_inodes=757538,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=610656k,mode=755)
/dev/sda1 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=33,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=1800)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
/dev/sdb1 on /storage type xfs (rw,noatime,attr2,inode64,noquota)
binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime)
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=610652k,mode=700,uid=1000,gid=1000)
gvfsd-fuse on /run/user/1000/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1000,group_id=1000)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=610652k,mode=700)

## We need to know exact kernel revision
root@ems01:/home/user/TIBCO_HOME# uname -a
Linux ems01 4.10.0-26-generic #30-Ubuntu SMP Tue Jun 27 09:30:12 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux

## We need to see syscall activities of EMS
root@ems01:/home/user/TIBCO_HOME# strace -ff -tt -yy -y -p `ps aux | grep tibemsd64 | grep -v bash | grep -v grep | awk '{print $2}'` 2>&1 | tee > ems_proc.txt
^C

#Ctrl+C after 5 minutes

## We need to know EMS stats
user@ems01:~/TIBCO_HOME/ems/8.3/bin$ ./tibemsadmin

TIBCO Enterprise Message Service Administration Tool.
Copyright 2003-2016 by TIBCO Software Inc.
All rights reserved.

Version 8.3.0 V14 3/9/2016

Type 'help' for commands help, 'exit' to exit:
>
> connect
Login name (admin):
Password: ***********
Connected to: tcp://localhost:7222
tcp://localhost:7222> info
Server: EMS-SERVER (version: 8.3.0 V14)
Process Id: 3875
State: active
Runtime Module Path: /home/user/TIBCO_HOME/ems/8.3/bin/lib/64:/home/user/TIBCO_HOME/ems/8.3/lib/64
Topics: 5 (0 dynamic, 0 temporary)
Queues: 10 (1 dynamic, 1 temporary)
Connections: 3
Sessions: 52
Producers: 2
Consumers: 1
Durables: 0
Log File Size: 3.3 Kb out of 10MB
Pending Messages: 63395
Pending Message Size: 92.2 MB
Message Memory Usage: 24.8 MB out of 2GB
Message Memory Pooled: 20.8 MB
Synchronous Storage: 26.0 Kb
Asynchronous Storage: 119.1 MB
Fsync for Sync Storage: disabled
Inbound Message Rate: 0 msgs/sec, 0.1 Kb per second
Outbound Message Rate: 0 msgs/sec, 0.1 Kb per second
Storage Read Rate: 0 reads/sec, 0.0 Kb per second
Storage Write Rate: 0 writes/sec, 0.0 Kb per second
Uptime: 156 days 19 hours 5 minutes

tcp://localhost:7222> show consumers full
Pre Msgs Sent Last Last Total Total
Id Conn Sess User T Queue SASB Pre Dlv Sent Size Uptime Sent Ackd Sent Acked
23774 6 22 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.426 946 - 50 0
23775 6 23 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.426 - - 0 0
23776 6 24 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.425 - - 0 0
23777 6 25 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.425 - - 0 0
23778 6 26 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.425 946 - 50 0
23779 6 27 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.425 - - 0 0
23780 6 28 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.424 - - 0 0
23781 6 29 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.424 - - 0 0
23782 6 30 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.424 946 - 50 0
23783 6 31 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.423 946 - 50 0
23784 6 32 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.423 - - 0 0
23785 6 33 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.423 946 - 50 0
23786 6 34 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.420 - - 0 0
23787 6 35 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.411 946 - 50 0
23788 6 36 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.410 946 - 50 0
23789 6 37 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.410 946 - 50 0
23790 6 38 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.410 - - 0 0
23791 6 39 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.410 946 - 50 0
23792 6 40 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23793 6 41 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23794 6 42 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23795 6 43 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23796 6 44 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23797 6 45 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23798 6 46 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.409 946 - 50 0
23799 6 47 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.408 946 - 50 0
23800 6 48 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.408 946 - 50 0
23801 6 49 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.408 946 - 50 0
23802 6 50 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 50 77K 0.408 946 - 50 0
23803 6 51 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 49 76K 0.408 946 1 50 1
23804 6 52 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 0 0 0 0.408 - - 0 0
23805 6 53 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 49 76K 0.407 946 1 50 1
23806 6 54 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 49 76K 0.407 946 1 50 1
23807 6 55 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 49 76K 0.407 946 1 50 1
23808 6 56 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 49 76K 0.407 946 1 50 1
23809 6 57 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C-- 50 50 49 76K 0.407 946 1 50 1
9 13 65 admin Q $TMP$.EMS-SERVER.F23596882A541.1 +N-- 5 1 0 0 0:08:21 492s 492s 3 3

tcp://localhost:7222> show stat producers
Total Count Rate/Second
User Conn T Destination Msgs Size Msgs Size
ecore.bus.transaction 6 - 0 0.0 Kb 0 0.0 Kb
admin 13 Q $sys.admin 6 1.0 Kb 0 0.0 Kb

tcp://localhost:7222> show connections
L ID FSXT S Host User ClientID Sess Uptime
J 6 ---Q + ems01 ecore.bus.transaction BW-JMS Connection-queue-ECoreIntegrationService-SubscriptionService 51 0:11:37
J 9 ---T + ems01 ecore.bus.transaction BW-JMS Connection-topic-ECoreIntegrationService-SubscriptionService 0 0:11:35
C 13 ---A + ems01 admin 1 0:11:27
tcp://localhost:7222>

tcp://localhost:7222> show db
Synchronous DB:
File Size: 2.0 Kb
Free Space: 1.0 Kb (48.2%)
Used Space: 1.0 Kb (50.0%)
Fragmentation: 100%
Message Count: 0
Message Size: 0.0 Kb

Asynchronous DB:
File Size: 119.1 MB
Free Space: 45.5 MB (38.2%)
Used Space: 73.6 MB (61.8%)
Fragmentation: 1%
Message Count: 39072
Message Size: 55.6 MB
tcp://localhost:7222>

## We need to know storage performance
root@ems01:/storage$ time dd if=/dev/zero bs=1M count=1024 of=test.dat && sync
1024+0 read records
1024+0 written records
1073741824 bytes (1,1 GB, 1,0 GiB), 1,00872 s, 1,1 GB/s

real 0m1,013s
user 0m0,000s
sys 0m0,600s

## run again to check repeatability

root@ems01:/storage$ time dd if=/dev/zero bs=1M count=1024 of=test.dat && sync
1024+0 read records
1024+0 written records
1073741824 bytes (1,1 GB, 1,0 GiB), 1,00482 s, 1,1 GB/s

real 0m1,184s
user 0m0,000s
sys 0m0,768s

## We need to know memory performance
root@ems01:/storage$ time dd if=/dev/zero bs=1M count=1024 of=/dev/null && sync
1024+0 read records
1024+0 written records
1073741824 bytes (1,1 GB, 1,0 GiB), 0,0854145 s, 12,6 GB/s

real 0m0,087s
user 0m0,000s
sys 0m0,072s

## We need to know fs performance
root@ems01:/storage$ time rm test.dat && sync

real 0m0,165s
user 0m0,000s
sys 0m0,152s

## We need to know fs performance
root@ems01:/storage$ time sync

real 0m0,006s
user 0m0,000s
sys 0m0,000s

## We need to know about DNS resolving
root@ems01:~# nslookup ems01
Server: 127.0.0.53
Address: 127.0.0.53#53

Non-authoritative answer:
Name: ems01
Address: 10.0.2.15

root@ems01:~# nslookup ems02
Server: 127.0.0.53
Address: 127.0.0.53#53

Non-authoritative answer:
Name: ems02
Address: 10.0.2.2

## We need to know about network topology
root@ems01:~# traceroute ems02
traceroute to ems02 (10.0.2.2), 30 hops max, 60 byte packets
1 ems02 (10.0.2.2) 2.245 ms 2.159 ms 2.114 ms

root@ems01:~# traceroute ems01
traceroute to ems01 (10.0.2.15), 30 hops max, 60 byte packets
1 ems01 (10.0.2.15) 0.012 ms 0.002 ms 0.001 ms

## We need to know about network performance
root@ems01:~# ping ems01
PING ems01 (10.0.2.15) 56(84) bytes of data.
64 bytes from ems01 (10.0.2.15): icmp_seq=1 ttl=64 time=0.013 ms
64 bytes from ems01 (10.0.2.15): icmp_seq=2 ttl=64 time=0.025 ms
64 bytes from ems01 (10.0.2.15): icmp_seq=3 ttl=64 time=0.019 ms
^C
--- ems01 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2026ms
rtt min/avg/max/mdev = 0.013/0.019/0.025/0.005 ms

root@ems01:~# ping ems02
PING ems02 (10.0.2.2) 56(84) bytes of data.
64 bytes from ems02 (10.0.2.2): icmp_seq=1 ttl=64 time=0.292 ms
64 bytes from ems02 (10.0.2.2): icmp_seq=2 ttl=64 time=0.493 ms
64 bytes from ems02 (10.0.2.2): icmp_seq=3 ttl=64 time=0.361 ms
64 bytes from ems02 (10.0.2.2): icmp_seq=4 ttl=64 time=0.220 ms
^C
--- ems02 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3063ms
rtt min/avg/max/mdev = 0.220/0.341/0.493/0.102 ms