Tracing Oracle connectivity problem (connection timed out on socket)

tcpdump -nnvvXS host oracle-db-prd

tcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes

11:33:13.433097 IP (tos 0x0, ttl 64, id 36655, offset 0, flags [DF], proto TCP (6), length 60)

    192.168.100.3.36283 > 192.168.1.10.1521: Flags [S], cksum 0x7cab (correct), seq 3238343368, win 14600, options [mss 1460,sackOK,TS val 678084071 ecr 0,nop,wscale 7], length 0

        0x0000:  4500 003c 8f2f 4000 4006 870c 0af3 025e  E..<./@.@......^

        0x0010:  0acb 0c65 8dbb 05f1 c105 32c8 0000 0000  ...e......2.....

        0x0020:  a002 3908 7cab 0000 0204 05b4 0402 080a  ..9.|...........

        0x0030:  286a bde7 0000 0000 0103 0307            (j..........

11:33:13.434046 IP (tos 0x0, ttl 63, id 0, offset 0, flags [DF], proto TCP (6), length 52)

    192.168.1.10.1521 > 192.168.100.3.36283: Flags [S.], cksum 0xacc3 (correct), seq 4037414683, ack 3238343369, win 5840, options [mss 1380,nop,nop,sackOK,nop,wscale 7], length 0

        0x0000:  4500 0034 0000 4000 3f06 1744 0acb 0c65  E..4..@.?..D...e

        0x0010:  0af3 025e 05f1 8dbb f0a6 0f1b c105 32c9  ...^..........2.

        0x0020:  8012 16d0 acc3 0000 0204 0564 0101 0402  ...........d....

        0x0030:  0103 0307                                ....

11:33:13.434153 IP (tos 0x0, ttl 64, id 36656, offset 0, flags [DF], proto TCP (6), length 40)

    192.168.100.3.36283 > 192.168.1.10.1521: Flags [.], cksum 0x03a3 (correct), seq 3238343369, ack 4037414684, win 115, length 0

        0x0000:  4500 0028 8f30 4000 4006 871f 0af3 025e  E..(.0@.@......^

        0x0010:  0acb 0c65 8dbb 05f1 c105 32c9 f0a6 0f1c  ...e......2.....

        0x0020:  5010 0073 03a3 0000                      P..s....

11:33:13.434506 IP (tos 0x0, ttl 64, id 36657, offset 0, flags [DF], proto TCP (6), length 321)

    192.168.100.3.36283 > 192.168.1.10.1521: Flags [P.], cksum 0x25b4 (incorrect -> 0x00d9), seq 3238343369:3238343650, ack 4037414684, win 115, length 281

        0x0000:  4500 0141 8f31 4000 4006 8605 0af3 025e  E..A.1@.@......^

        0x0010:  0acb 0c65 8dbb 05f1 c105 32c9 f0a6 0f1c  ...e......2.....

        0x0020:  5018 0073 25b4 0000 0119 0000 0100 0000  P..s%...........

        0x0030:  0136 012c 0e41 2000 7fff 4f98 0000 0001  .6.,.A....O.....

        0x0040:  00df 003a 0000 0000 8181 0000 0000 0000  ...:............

        0x0050:  0000 0000 0000 0000 0000 0000 0000 0000  ................

        0x0060:  0000 2844 4553 4352 4950 5449 4f4e 3d28  ..(DESCRIPTION=(

        0x0070:  4144 4452 4553 533d 2850 524f 544f 434f  ADDRESS=(PROTOCO

        0x0080:  4c3d 5443 5029 2848 4f53 543d 3130 2e32  L=TCP)(HOST=orac

        0x0090:  3033 2e31 322e 3130 3129 2850 4f52 543d  le-db-prd)(PORT=

        0x00a0:  3135 3231 2929 2843 4f4e 4e45 4354 5f44  1521))(CONNECT_D

        0x00b0:  4154 413d 2843 4944 3d28 5052 4f47 5241  ATA=(CID=(PROGRA

        0x00c0:  4d3d 4a44 4243 2054 6869 6e20 436c 6965  M=JDBC.Thin.Clie

        0x00d0:  6e74 2928 484f 5354 3d5f 5f6a 6462 635f  nt)(HOST=__jdbc_

        0x00e0:  5f29 2855 5345 523d 7469 6263 6f29 2928  _)(USER=tibco))(

        0x00f0:  5345 5256 4943 455f 4e41 4d45 3d71 786c  SERVICE_NAME=crm

        0x0100:  6477 6829 2843 4944 3d28 5052 4f47 5241  001)(CID=(PROGRA

        0x0110:  4d3d 4a44 4243 2054 6869 6e20 436c 6965  M=JDBC.Thin.Clie

        0x0120:  6e74 2928 484f 5354 3d5f 5f6a 6462 635f  nt)(HOST=__jdbc_

        0x0130:  5f29 2855 5345 523d 7469 6263 6f29 2929  _)(USER=tibco)))

        0x0140:  29                                       )

11:33:13.435530 IP (tos 0x0, ttl 63, id 15716, offset 0, flags [DF], proto TCP (6), length 40)

    192.168.1.10.1521 > 192.168.100.3.36283: Flags [.], cksum 0x02c7 (correct), seq 4037414684, ack 3238343650, win 54, length 0

        0x0000:  4500 0028 3d64 4000 3f06 d9eb 0acb 0c65  E..(=d@.?......e

        0x0010:  0af3 025e 05f1 8dbb f0a6 0f1c c105 33e2  ...^..........3.

        0x0020:  5010 0036 02c7 0000 0000 0000 0000       P..6..........

11:33:13.435660 IP (tos 0x0, ttl 63, id 15717, offset 0, flags [DF], proto TCP (6), length 117)

    192.168.1.10.1521 > 192.168.100.3.36283: Flags [P.], cksum 0x8f72 (correct), seq 4037414684:4037414761, ack 3238343650, win 54, length 77

        0x0000:  4500 0075 3d65 4000 3f06 d99d 0acb 0c65  E..u=e@.?......e

        0x0010:  0af3 025e 05f1 8dbb f0a6 0f1c c105 33e2  ...^..........3.

        0x0020:  5018 0036 8f72 0000 004d 0000 0500 0000  P..6.r...M......

        0x0030:  0043 2844 4553 4352 4950 5449 4f4e 3d28  .C(DESCRIPTION=(

        0x0040:  4144 4452 4553 533d 2850 524f 544f 434f  ADDRESS=(PROTOCO

        0x0050:  4c3d 5443 5029 2848 4f53 543d 3130 2e32  L=TCP)(HOST=orac

        0x0060:  3033 2e31 322e 3429 2850 4f52 543d 3135  le-prd1)(PORT=15

        0x0070:  3231 2929 29                             21)))

11:33:13.435682 IP (tos 0x0, ttl 64, id 36658, offset 0, flags [DF], proto TCP (6), length 40)

    192.168.100.3.36283 > 192.168.1.10.1521: Flags [.], cksum 0x023d (correct), seq 3238343650, ack 4037414761, win 115, length 0

        0x0000:  4500 0028 8f32 4000 4006 871d 0af3 025e  E..(.2@.@......^

        0x0010:  0acb 0c65 8dbb 05f1 c105 33e2 f0a6 0f69  ...e......3....i

        0x0020:  5010 0073 023d 0000                      P..s.=..

11:33:13.435700 IP (tos 0x0, ttl 63, id 15718, offset 0, flags [DF], proto TCP (6), length 40)

    192.168.1.10.1521 > 192.168.100.3.36283: Flags [F.], cksum 0x0279 (correct), seq 4037414761, ack 3238343650, win 54, length 0

        0x0000:  4500 0028 3d66 4000 3f06 d9e9 0acb 0c65  E..(=f@.?......e

        0x0010:  0af3 025e 05f1 8dbb f0a6 0f69 c105 33e2  ...^.......i..3.

        0x0020:  5011 0036 0279 0000 0000 0000 0000       P..6.y........

11:33:13.436176 IP (tos 0x0, ttl 64, id 36659, offset 0, flags [DF], proto TCP (6), length 40)

    192.168.100.3.36283 > 192.168.1.10.1521: Flags [F.], cksum 0x023b (correct), seq 3238343650, ack 4037414762, win 115, length 0

        0x0000:  4500 0028 8f33 4000 4006 871c 0af3 025e  E..(.3@.@......^

        0x0010:  0acb 0c65 8dbb 05f1 c105 33e2 f0a6 0f6a  ...e......3....j

        0x0020:  5011 0073 023b 0000                      P..s.;..

11:33:13.437033 IP (tos 0x0, ttl 63, id 15719, offset 0, flags [DF], proto TCP (6), length 40)

    192.168.1.10.1521 > 192.168.100.3.36283: Flags [.], cksum 0x0278 (correct), seq 4037414762, ack 3238343651, win 54, length 0

        0x0000:  4500 0028 3d67 4000 3f06 d9e8 0acb 0c65  E..(=g@.?......e

        0x0010:  0af3 025e 05f1 8dbb f0a6 0f6a c105 33e3  ...^.......j..3.

        0x0020:  5010 0036 0278 0000 0000 0000 0000       P..6.x........

JDBC driver wants to connect to oracle-db-prd, which is a Oracle grid dispatcher and redirects to oracle-prd1, but this host is blocked by firewall and from application you see:

caused by: java.sql.SQLRecoverableException: IO Error: The Network Adapter could not establish the connection
        at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:458)
        at oracle.jdbc.driver.PhysicalConnection.(PhysicalConnection.java:546)
        at oracle.jdbc.driver.T4CConnection.(T4CConnection.java:236)
        at oracle.jdbc.driver.T4CDriverExtension.getConnection(T4CDriverExtension.java:32)
        at oracle.jdbc.driver.OracleDriver.connect(OracleDriver.java:521)
        at java.sql.DriverManager.getConnection(Unknown Source)
        at java.sql.DriverManager.getConnection(Unknown Source)
        at com.tibco.pe.core.JDBCPool.getConnectionEntry(Unknown Source)
        at com.tibco.pe.core.JDBCPool.getConnectionEntry(Unknown Source)
        at com.tibco.pe.core.JDBCPoolManager.getConnectionEntry(Unknown Source)
        at com.tibco.plugin.jdbc.JDBCActivity.eval(Unknown Source)
        at com.tibco.pe.plugin.Activity.eval(Unknown Source)
        at com.tibco.pe.core.TaskImpl.eval(Unknown Source)
        at com.tibco.pe.core.Job.a(Unknown Source)
        at com.tibco.pe.core.Job.k(Unknown Source)
        at com.tibco.pe.core.JobDispatcher$JobCourier.a(Unknown Source)
        at com.tibco.pe.core.JobDispatcher$JobCourier.run(Unknown Source)
Caused by: oracle.net.ns.NetException: The Network Adapter could not establish the connection
        at oracle.net.nt.ConnStrategy.execute(ConnStrategy.java:392)
        at oracle.net.resolver.AddrResolution.resolveAndExecute(AddrResolution.java:434)
        at oracle.net.ns.NSProtocol.establishConnection(NSProtocol.java:687)
        at oracle.net.ns.NSProtocol.connect(NSProtocol.java:343)
        at oracle.jdbc.driver.T4CConnection.connect(T4CConnection.java:1102)
        at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:320)
        ... 16 more
Caused by: java.net.SocketTimeoutException: connect timed out
        at java.net.PlainSocketImpl.socketConnect(Native Method)