We are running the application on 20g jvm with Object Cache assigned 10g memory.
We are getting OOM error at times
Below are the datomic metrics .Strangely , AvailableMB get reduced from 3850 to 2090 in 2 mins.
Any pointers will be helpful
thanks
2019-06-27_19:26:01.251 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:tid 25, :ObjectCacheCount 25096, :PeerAcceptNewMsec {:lo 0, :
hi 0, :sum 0, :count 21}, :AvailableMB 3850.0, :StorageGetMsec {:lo 1, :hi 38, :sum 103, :count 14}, :pid 7417, :event :metrics, :ObjectCache {:lo 0, :hi
1, :sum 63330, :count 63348}, :MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerFulltextBatch {:lo 1, :hi 1, :sum 21, :count 21}, :StorageGetBytes {:l
o 4025, :hi 30983, :sum 284471, :count 14}}
2019-06-27_19:27:03.385 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerAcceptNe
wMsec {:lo 0, :hi 0, :sum 0, :count 32}, :PeerFulltextBatch {:lo 1, :hi 6, :sum 32, :count 27}, :ObjectCache {:lo 1, :hi 1, :sum 242, :count 242}, :Availa
bleMB 3300.0, :ObjectCacheCount 25096, :event :metrics, :pid 7417, :tid 25}
2019-06-27_19:28:06.566 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerAcceptNe
wMsec {:lo 0, :hi 0, :sum 0, :count 35}, :PeerFulltextBatch {:lo 1, :hi 6, :sum 35, :count 28}, :AvailableMB 2090.0, :ObjectCacheCount 25096, :event :metr
ics, :pid 7417, :tid 25}
2019-06-27_19:29:14.753 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerAcceptNe
wMsec {:lo 0, :hi 81, :sum 152, :count 95}, :PeerFulltextBatch {:lo 1, :hi 6, :sum 95, :count 88}, :ObjectCache {:lo 1, :hi 1, :sum 242, :count 242}, :Ava
ilableMB 1470.0, :ObjectCacheCount 25096, :event :metrics, :pid 7417, :tid 25}
2019-06-27_19:29:14.788 default [clojure-agent-send-off-pool-3] INFO datomic.db - {:event :db/accept-index, :basis-t :basisT, :phase :begin, :pid 7417, :
tid 41}
2019-06-27_19:30:10.748 default [clojure-agent-send-off-pool-3] INFO datomic.db - {:event :db/accept-index, :basis-t :basisT, :msec 56000.0, :phase :end,
:pid 7417, :tid 41}
2019-06-27_19:30:26.253 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:tid 25, :ObjectCacheCount 25113, :PeerAcceptNewMsec {:lo 0, :
hi 0, :sum 0, :count 91}, :AvailableMB 1320.0, :StorageGetMsec {:lo 0, :hi 5, :sum 43, :count 31}, :pid 7417, :event :metrics, :ObjectCache {:lo 0, :hi 1,
:sum 1, :count 32}, :MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerFulltextBatch {:lo 1, :hi 2, :sum 91, :count 90}, :StorageGetBytes {:lo 69, :hi
83855, :sum 804572, :count 31}, :AcceptIndexMsec {:lo 56000, :hi 56000, :sum 56000, :count 1}}
2019-06-27_19:31:40.491 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:tid 25, :ObjectCacheCount 25125, :PeerAcceptNewMsec {:lo 0, :
hi 19, :sum 53, :count 120}, :AvailableMB 1120.0, :StorageGetMsec {:lo 0, :hi 7, :sum 89, :count 32}, :pid 7417, :event :metrics, :ObjectCache {:lo 0, :hi
1, :sum 726, :count 764}, :MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerFulltextBatch {:lo 1, :hi 14, :sum 120, :count 106}, :StorageGetBytes {:l
o 93, :hi 56231, :sum 1042126, :count 32}}
2019-06-27_19:32:51.599 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerAcceptNe
wMsec {:lo 0, :hi 0, :sum 0, :count 92}, :PeerFulltextBatch {:lo 1, :hi 1, :sum 92, :count 92}, :AvailableMB 1030.0, :ObjectCacheCount 25125, :event :metr
ics, :pid 7417, :tid 25}
2019-06-27_19:33:59.649 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerAcceptNe
wMsec {:lo 0, :hi 0, :sum 0, :count 63}, :PeerFulltextBatch {:lo 1, :hi 2, :sum 63, :count 62}, :ObjectCache {:lo 1, :hi 1, :sum 676, :count 676}, :Availa
bleMB 997.0, :ObjectCacheCount 25125, :event :metrics, :pid 7417, :tid 25}
2019-06-27_19:35:11.719 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:PeerAcceptNewMsec {:lo 0, :hi 21600, :sum 21600, :count 110},
:PeerFulltextBatch {:lo 1, :hi 21, :sum 110, :count 70}, :MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :AvailableMB 988.0, :ObjectCacheCount 25125, :e
vent :metrics, :pid 7417, :tid 25}
2019-06-27_19:36:12.991 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:PeerAcceptNewMsec {:lo 0, :hi 0, :sum 0, :count 77}, :Metrics
Report {:lo 1, :hi 1, :sum 1, :count 1}, :PeerFulltextBatch {:lo 1, :hi 5, :sum 77, :count 69}, :AvailableMB 984.0, :ObjectCacheCount 25125, :event :metri
cs, :pid 7417, :tid 25}
2019-06-27_19:37:14.843 default [Datomic Metrics Reporter] INFO datomic.process-monitor - {:MetricsReport {:lo 1, :hi 1, :sum 1, :count 1}, :PeerAcceptNe
wMsec {:lo 0, :hi 12300, :sum 12300, :count 70}, :PeerFulltextBatch {:lo 1, :hi 1, :sum 69, :count 69}, :AvailableMB 982.0, :ObjectCacheCount 25125, :event :metrics, :pid 7417, :tid 25}
java.lang.OutOfMemoryError: GC overhead limit exceeded
java.lang.OutOfMemoryError: GC overhead limit exceeded