piątek, lipca 14, 2017

Analysing EMS performace

## We need to know kernel log
root@ems01:/home/user/TIBCO_HOME# dmesg > dmesg.txt

## We need to what's going on in memory subsystem
root@ems01:/home/user/TIBCO_HOME# vmstat
procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 2  0      0 2573920 225408 1240940    0    0   874   374  677 9536 12  3 84  1  0

## We need to what's going on with I/O
root@ems01:/home/user/TIBCO_HOME# iostat
Linux 4.10.0-26-generic (ems01) 07/14/17 _x86_64_ (2 CPU)

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
          10.65    1.01    3.49    0.72    0.00   84.14

Device:            tps    kB_read/s    kB_wrtn/s    kB_read    kB_wrtn
sda              92.05      1519.83       587.98    1065413     412180
sdb               1.36       184.28       141.54     129180      99224

## We need to know how filesystems are mounted
root@ems01:/home/user/TIBCO_HOME# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
udev on /dev type devtmpfs (rw,nosuid,relatime,size=3030152k,nr_inodes=757538,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=610656k,mode=755)
/dev/sda1 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=33,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=1800)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime)
/dev/sdb1 on /storage type xfs (rw,noatime,attr2,inode64,noquota)
binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime)
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=610652k,mode=700,uid=1000,gid=1000)
gvfsd-fuse on /run/user/1000/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1000,group_id=1000)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=610652k,mode=700)

## We need to know exact kernel revision
root@ems01:/home/user/TIBCO_HOME# uname -a
Linux ems01 4.10.0-26-generic #30-Ubuntu SMP Tue Jun 27 09:30:12 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux

## We need to see syscall activities of EMS
root@ems01:/home/user/TIBCO_HOME# strace -ff -tt -yy -y -p `ps aux | grep tibemsd64 | grep -v bash | grep -v grep | awk '{print $2}'` 2>&1 | tee > ems_proc.txt
^C

#Ctrl+C after 5 minutes

## We need to know EMS stats
user@ems01:~/TIBCO_HOME/ems/8.3/bin$ ./tibemsadmin

TIBCO Enterprise Message Service Administration Tool.
Copyright 2003-2016 by TIBCO Software Inc.
All rights reserved.

Version 8.3.0 V14 3/9/2016

Type 'help' for commands help, 'exit' to exit:
>
> connect
Login name (admin):
Password: ***********
Connected to: tcp://localhost:7222
tcp://localhost:7222> info
 Server:                   EMS-SERVER (version: 8.3.0 V14)
 Process Id:               3875
 State:                    active
 Runtime Module Path:      /home/user/TIBCO_HOME/ems/8.3/bin/lib/64:/home/user/TIBCO_HOME/ems/8.3/lib/64
 Topics:                   5 (0 dynamic, 0 temporary)
 Queues:                   10 (1 dynamic, 1 temporary)
 Connections:              3
 Sessions:                 52
 Producers:                2
 Consumers:                1
 Durables:                 0
 Log File Size:            3.3 Kb out of 10MB
 Pending Messages:         63395
 Pending Message Size:     92.2 MB
 Message Memory Usage:     24.8 MB out of 2GB
 Message Memory Pooled:    20.8 MB
 Synchronous Storage:      26.0 Kb
 Asynchronous Storage:     119.1 MB
 Fsync for Sync Storage:   disabled
 Inbound Message Rate:     0 msgs/sec,  0.1 Kb per second
 Outbound Message Rate:    0 msgs/sec,  0.1 Kb per second
 Storage Read Rate:        0 reads/sec,  0.0 Kb per second
 Storage Write Rate:       0 writes/sec, 0.0 Kb per second
 Uptime:                   156 days 19 hours 5 minutes

tcp://localhost:7222> show consumers full
                                                                                     Pre Msgs Sent          Last Last Total Total
   Id Conn Sess User          T Queue                                        SASB Pre Dlv Sent Size   Uptime Sent Ackd  Sent Acked
23774    6   22 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.426  946    -    50     0
23775    6   23 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.426    -    -     0     0
23776    6   24 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.425    -    -     0     0
23777    6   25 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.425    -    -     0     0
23778    6   26 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.425  946    -    50     0
23779    6   27 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.425    -    -     0     0
23780    6   28 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.424    -    -     0     0
23781    6   29 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.424    -    -     0     0
23782    6   30 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.424  946    -    50     0
23783    6   31 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.423  946    -    50     0
23784    6   32 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.423    -    -     0     0
23785    6   33 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.423  946    -    50     0
23786    6   34 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.420    -    -     0     0
23787    6   35 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.411  946    -    50     0
23788    6   36 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.410  946    -    50     0
23789    6   37 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.410  946    -    50     0
23790    6   38 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.410    -    -     0     0
23791    6   39 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.410  946    -    50     0
23792    6   40 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23793    6   41 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23794    6   42 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23795    6   43 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23796    6   44 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23797    6   45 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23798    6   46 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.409  946    -    50     0
23799    6   47 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.408  946    -    50     0
23800    6   48 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.408  946    -    50     0
23801    6   49 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.408  946    -    50     0
23802    6   50 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   50  77K    0.408  946    -    50     0
23803    6   51 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   49  76K    0.408  946    1    50     1
23804    6   52 ecore.bus.transaction q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50   0    0    0    0.408    -    -     0     0
23805    6   53 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   49  76K    0.407  946    1    50     1
23806    6   54 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   49  76K    0.407  946    1    50     1
23807    6   55 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   49  76K    0.407  946    1    50     1
23808    6   56 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   49  76K    0.407  946    1    50     1
23809    6   57 ecore.bus.transaction Q ecore.BUS.TRANSACTION.TO.LOG.QUEUE +C--  50  50   49  76K    0.407  946    1    50     1
    9   13   65 admin         Q $TMP$.EMS-SERVER.F23596882A541.1             +N--   5   1    0    0  0:08:21 492s 492s     3     3

tcp://localhost:7222> show stat producers
                                          Total Count      Rate/Second
User           Conn  T  Destination       Msgs   Size      Msgs   Size
ecore.bus.transaction     6  -                       0    0.0 Kb      0    0.0 Kb
admin            13  Q  $sys.admin           6    1.0 Kb      0    0.0 Kb

tcp://localhost:7222> show connections
L  ID  FSXT  S  Host            User          ClientID                                                                 Sess     Uptime
J  6   ---Q  +  ems01 ecore.bus.transaction BW-JMS Connection-queue-ECoreIntegrationService-SubscriptionService   51    0:11:37
J  9   ---T  +  ems01 ecore.bus.transaction BW-JMS Connection-topic-ECoreIntegrationService-SubscriptionService    0    0:11:35
C  13  ---A  +  ems01 admin                                                                                     1    0:11:27
tcp://localhost:7222>

tcp://localhost:7222> show db
Synchronous DB:
 File Size:                     2.0 Kb
 Free Space:                    1.0 Kb (48.2%)
 Used Space:                    1.0 Kb (50.0%)
 Fragmentation:                 100%
 Message Count:                 0
 Message Size:                  0.0 Kb

Asynchronous DB:
 File Size:                     119.1 MB
 Free Space:                    45.5 MB (38.2%)
 Used Space:                    73.6 MB (61.8%)
 Fragmentation:                 1%
 Message Count:                 39072
 Message Size:                  55.6 MB
tcp://localhost:7222>

## We need to know storage performance
root@ems01:/storage$ time dd if=/dev/zero bs=1M count=1024 of=test.dat && sync
1024+0 read records
1024+0 written records
1073741824 bytes (1,1 GB, 1,0 GiB), 1,00872 s, 1,1 GB/s

real 0m1,013s
user 0m0,000s
sys 0m0,600s

## run again to check repeatability

root@ems01:/storage$ time dd if=/dev/zero bs=1M count=1024 of=test.dat && sync
1024+0 read records
1024+0 written records
1073741824 bytes (1,1 GB, 1,0 GiB), 1,00482 s, 1,1 GB/s

real 0m1,184s
user 0m0,000s
sys 0m0,768s

## We need to know memory performance
root@ems01:/storage$ time dd if=/dev/zero bs=1M count=1024 of=/dev/null && sync
1024+0 read records
1024+0 written records
1073741824 bytes (1,1 GB, 1,0 GiB), 0,0854145 s, 12,6 GB/s

real 0m0,087s
user 0m0,000s
sys 0m0,072s

## We need to know fs performance
root@ems01:/storage$ time rm test.dat && sync

real 0m0,165s
user 0m0,000s
sys 0m0,152s

## We need to know fs performance
root@ems01:/storage$ time sync

real 0m0,006s
user 0m0,000s
sys 0m0,000s

## We need to know about DNS resolving
root@ems01:~# nslookup ems01
Server: 127.0.0.53
Address: 127.0.0.53#53

Non-authoritative answer:
Name: ems01
Address: 10.0.2.15

root@ems01:~# nslookup ems02
Server: 127.0.0.53
Address: 127.0.0.53#53

Non-authoritative answer:
Name: ems02
Address: 10.0.2.2

## We need to know about network topology
root@ems01:~# traceroute ems02
traceroute to ems02 (10.0.2.2), 30 hops max, 60 byte packets
 1  ems02 (10.0.2.2)  2.245 ms  2.159 ms  2.114 ms

root@ems01:~# traceroute ems01
traceroute to ems01 (10.0.2.15), 30 hops max, 60 byte packets
 1  ems01 (10.0.2.15)  0.012 ms  0.002 ms  0.001 ms

## We need to know about network performance
root@ems01:~# ping ems01
PING ems01 (10.0.2.15) 56(84) bytes of data.
64 bytes from ems01 (10.0.2.15): icmp_seq=1 ttl=64 time=0.013 ms
64 bytes from ems01 (10.0.2.15): icmp_seq=2 ttl=64 time=0.025 ms
64 bytes from ems01 (10.0.2.15): icmp_seq=3 ttl=64 time=0.019 ms
^C
--- ems01 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2026ms
rtt min/avg/max/mdev = 0.013/0.019/0.025/0.005 ms

root@ems01:~# ping ems02
PING ems02 (10.0.2.2) 56(84) bytes of data.
64 bytes from ems02 (10.0.2.2): icmp_seq=1 ttl=64 time=0.292 ms
64 bytes from ems02 (10.0.2.2): icmp_seq=2 ttl=64 time=0.493 ms
64 bytes from ems02 (10.0.2.2): icmp_seq=3 ttl=64 time=0.361 ms
64 bytes from ems02 (10.0.2.2): icmp_seq=4 ttl=64 time=0.220 ms
^C
--- ems02 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 3063ms
rtt min/avg/max/mdev = 0.220/0.341/0.493/0.102 ms

0 komentarze: