OutOfMemoryError on both sides peer and transactor

Datomic Pro Troubleshooting
1

Hi datomic team, are there any ideas what might happen when peer and transaction indicates about OutOfMemoryError, according lo the logs below I see it it somehow related to indexing, we started to get these errors recently on several different envs with very small traffic, restarting of the peer and the transacrtor do not help, after some time we see these errors again. datomic version is 1.0.7277

Peer logs :

clojure.lang.ExceptionInfo: Unable to convert data: {:key "67c05464-8a44-4e69-b413-48da3561370a", :crc32 3420686353, :size 22731791}
	at datomic.cache$report_val_fn_fail.invokeStatic(cache.clj:69)
	at datomic.cache$report_val_fn_fail.invoke(cache.clj:65)
	at datomic.domain.DeserializingRepairingLookup$fn__16804.invoke(domain.clj:217)
	at datomic.domain.DeserializingRepairingLookup.valAt(domain.clj:214)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.cache$lookup_with_inflight_cache$reify__9410$fn__9411.invoke(cache.clj:184)
	at clojure.lang.Delay.deref(Delay.java:42)
	at clojure.core$deref.invokeStatic(core.clj:2337)
	at clojure.core$deref.invoke(core.clj:2323)
	at datomic.cache$lookup_with_inflight_cache$reify__9410.valAt(cache.clj:187)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.cache$lookup_cache$reify__9407.valAt(cache.clj:151)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.common$getx.invokeStatic(common.clj:209)
	at datomic.common$getx.invoke(common.clj:205)
	at datomic.index$get_dir_node.invokeStatic(index.clj:194)
	at datomic.index$get_dir_node.invoke(index.clj:185)
	at datomic.index.Index.seek(index.clj:566)
	at datomic.db$ident_setting_datoms.invokeStatic(db.clj:2759)
	at datomic.db$ident_setting_datoms.invoke(db.clj:2752)
	at datomic.db$run_hooks.invokeStatic(db.clj:2774)
	at datomic.db$run_hooks.invoke(db.clj:2765)
	at datomic.db$db.invokeStatic(db.clj:2864)
	at datomic.db$db.invoke(db.clj:2848)
	at datomic.db_io$load_db_from_basis.invokeStatic(db_io.clj:75)
	at datomic.db_io$load_db_from_basis.doInvoke(db_io.clj:69)
	at clojure.lang.RestFn.invoke(RestFn.java:467)
	at datomic.peer.Connection$fn__21505$load_db__21513.invoke(peer.clj:225)
	at datomic.peer.Connection$fn__21505.invoke(peer.clj:226)
	at datomic.peer.Connection.create_connection_state(peer.clj:204)
	at datomic.peer$create_connection$reconnect_fn__21588.invoke(peer.clj:488)
	at clojure.core$partial$fn__5908.invoke(core.clj:2640)
	at datomic.common$retry_fn$fn__9168.invoke(common.clj:545)
	at datomic.common$retry_fn.invokeStatic(common.clj:545)
	at datomic.common$retry_fn.doInvoke(common.clj:528)
	at clojure.lang.RestFn.invoke(RestFn.java:713)
	at datomic.peer$create_connection$fn__21590.invoke(peer.clj:492)
	at datomic.reconnector2.Reconnector$fn__17136.invoke(reconnector2.clj:57)
	at clojure.core$binding_conveyor_fn$fn__5823.invoke(core.clj:2047)
	at clojure.lang.AFn.call(AFn.java:18)
	at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.base/java.lang.Thread.run(Unknown Source)
Caused by: java.lang.OutOfMemoryError: Java heap space

Transactor logs:

2025-02-27 12:00:04.165 WARN  default    datomic.index - {:message "merge-one-index failed", :pid 1, :tid 5814}
java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError: Java heap space
	at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:191)
	at clojure.core$deref_future.invokeStatic(core.clj:2317)
	at clojure.core$future_call$reify__8544.deref(core.clj:7042)
	at clojure.core$deref.invokeStatic(core.clj:2337)
	at clojure.core$deref.invoke(core.clj:2323)
	at clojure.core$map$fn__5935.invoke(core.clj:2772)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at clojure.lang.LazySeq.seq(LazySeq.java:58)
	at clojure.lang.RT.seq(RT.java:535)
	at clojure.core$seq__5467.invokeStatic(core.clj:139)
	at clojure.core$seq__5467.invoke(core.clj:139)
	at datomic.index$merge_one_index$fn__15019.invoke(index.clj:1377)
	at datomic.index$merge_one_index.invokeStatic(index.clj:1377)
	at datomic.index$merge_one_index.invoke(index.clj:1220)
	at datomic.index$merge_db_STAR_$build_tiered_index__15285$build_index__15289.invoke(index.clj:1814)
	at datomic.index$merge_db_STAR_$build_tiered_index__15285$fn__15335.invoke(index.clj:1897)
	at datomic.index$merge_db_STAR_$build_tiered_index__15285.invoke(index.clj:1885)
	at datomic.index$merge_db_STAR_.invokeStatic(index.clj:1909)
	at datomic.index$merge_db_STAR_.invoke(index.clj:1796)
	at datomic.index$merge_db$fn__15387$fn__15391.invoke(index.clj:2014)
	at datomic.index$merge_db$fn__15387.invoke(index.clj:2007)
	at datomic.index$merge_db.invokeStatic(index.clj:1998)
	at datomic.index$merge_db.invoke(index.clj:1988)
	at datomic.update$process_request_index$fn__27023$fn__27024$fn__27028.invoke(update.clj:315)
	at datomic.update$process_request_index$fn__27023$fn__27024.invoke(update.clj:310)
	at datomic.update$process_request_index$fn__27023.invoke(update.clj:306)
	at clojure.core$binding_conveyor_fn$fn__5823.invoke(core.clj:2047)
	at clojure.lang.AFn.call(AFn.java:18)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.OutOfMemoryError: Java heap space
	at java.base/java.util.Arrays.copyOf(Arrays.java:3536)
	at java.base/java.io.ByteArrayOutputStream.ensureCapacity(ByteArrayOutputStream.java:100)
	at java.base/java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:130)
	at java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:123)
	at java.base/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
	at org.fressian.impl.RawOutput.writeRawBytes(RawOutput.java:95)
	at org.fressian.FressianWriter.writeString(FressianWriter.java:138)
	at org.fressian.impl.Handlers$6.write(Handlers.java:68)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at org.fressian.impl.Handlers$13.write(Handlers.java:139)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at datomic.index$reify__14764.write(index.clj:606)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at datomic.index$reify__14768.write(index.clj:625)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at datomic.fressian$fressian.invokeStatic(fressian.clj:87)
	at datomic.fressian$fressian.doInvoke(fressian.clj:76)
	at clojure.lang.RestFn.applyTo(RestFn.java:142)
	at clojure.core$apply.invokeStatic(core.clj:671)
	at clojure.core$apply.invoke(core.clj:662)
2025-02-27 12:00:04.165 WARN  default    datomic.index - ... caused by ...
java.lang.OutOfMemoryError: Java heap space
	at java.base/java.util.Arrays.copyOf(Arrays.java:3536)
	at java.base/java.io.ByteArrayOutputStream.ensureCapacity(ByteArrayOutputStream.java:100)
	at java.base/java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:130)
	at java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:123)
	at java.base/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73)
	at org.fressian.impl.RawOutput.writeRawBytes(RawOutput.java:95)
	at org.fressian.FressianWriter.writeString(FressianWriter.java:138)
	at org.fressian.impl.Handlers$6.write(Handlers.java:68)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at org.fressian.impl.Handlers$13.write(Handlers.java:139)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at datomic.index$reify__14764.write(index.clj:606)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at datomic.index$reify__14768.write(index.clj:625)
	at org.fressian.FressianWriter.doWrite(FressianWriter.java:449)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:459)
	at org.fressian.FressianWriter.writeAs(FressianWriter.java:464)
	at org.fressian.FressianWriter.writeObject(FressianWriter.java:472)
	at datomic.fressian$fressian.invokeStatic(fressian.clj:87)
	at datomic.fressian$fressian.doInvoke(fressian.clj:76)
	at clojure.lang.RestFn.applyTo(RestFn.java:142)
	at clojure.core$apply.invokeStatic(core.clj:671)
	at clojure.core$apply.invoke(core.clj:662)
2

We found the root cause, both envs where the problem occurs were affected by the transaction where a bunch of attributes were affected by attribute :db/noHistory=true, and right after this the transactor started to notify failed indexing jobs, on the screen you can see the time when :db/noHistory=true was added and checking the transactor logs you can see when the first failed indexing job happened. @jaret could you please tell whether adding noHistory=true to attributes triggers indexing on the trasactor side?

The screen from datomic console, the transaction hapeened at 13:18:07
The query was:

[:find ?ident ?tx ?date ?added
 :where
 [?e :db/noHistory ?doc ?tx ?added]
 [?e :db/ident ?ident]
 [?tx :db/txInstant ?date]
]

image
image929×594 122 KB

The transactor logs, the first error happened at 13:18:15:

java.lang.OutOfMemoryError: Java heap space
2025-02-19 13:18:15.856 WARN  default    datomic.index - ... caused by ...
	at clojure.lang.LazySeq.seq(LazySeq.java:51)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at clojure.core$map$fn__5935.invoke(core.clj:2772)
	at datomic.index$merge_db_STAR_$build_tiered_index__13838$fn__13888$fn__13898.invoke(index.clj:2050)
	at datomic.cache$getx_uncached.invoke(cache.clj:57)
	at datomic.cache$getx_uncached.invokeStatic(cache.clj:59)
	at datomic.cache$lookup_cache$reify__11891.valAtUncached(cache.clj:169)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.cache$lookup_with_inflight_cache$reify__11894.valAt(cache.clj:187)
	at clojure.core$deref.invoke(core.clj:2323)
	at clojure.core$deref.invokeStatic(core.clj:2337)
	at clojure.lang.Delay.deref(Delay.java:42)
	at datomic.cache$lookup_with_inflight_cache$reify__11894$fn__11895.invoke(cache.clj:184)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.domain.DeserializingRepairingLookup.valAt(domain.clj:214)
	at datomic.domain.DeserializingRepairingLookup$fn__17950.invoke(domain.clj:215)
	at datomic.domain$deserialize.invoke(domain.clj:195)
	at datomic.domain$deserialize.invokeStatic(domain.clj:198)
	at datomic.fressian$val__GT_obj$fn__9674.invoke(fressian.clj:223)
	at datomic.fressian$val__GT_obj$fn__9674$fn__9675.invoke(fressian.clj:228)
	at org.fressian.FressianReader.readObject(FressianReader.java:274)
	at org.fressian.FressianReader.read(FressianReader.java:666)
	at org.fressian.FressianReader.handleStruct(FressianReader.java:695)
	at datomic.index$reify__13311.read(index.clj:635)
	at org.fressian.FressianReader.readObject(FressianReader.java:274)
	at org.fressian.FressianReader.read(FressianReader.java:623)
	at org.fressian.FressianReader.readClosedList(FressianReader.java:827)
	at org.fressian.FressianReader.read(FressianReader.java:604)
	at org.fressian.FressianReader.internalReadChunkedString(FressianReader.java:752)
	at java.base/java.lang.StringBuilder.toString(StringBuilder.java:450)
	at java.base/java.lang.StringUTF16.newString(StringUTF16.java:1179)
	at java.base/java.util.Arrays.copyOfRange(Arrays.java:3821)
Caused by: java.lang.OutOfMemoryError: Java heap space
	at java.base/java.lang.Thread.run(Thread.java:831)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1135)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at clojure.lang.AFn.call(AFn.java:18)
	at clojure.core$binding_conveyor_fn$fn__5823.invoke(core.clj:2047)
	at datomic.update$process_request_index$fn__25056.invoke(update.clj:306)
	at datomic.update$process_request_index$fn__25056$fn__25057.invoke(update.clj:310)
	at datomic.update$process_request_index$fn__25056$fn__25057$fn__25061.invoke(update.clj:315)
	at datomic.index$merge_db.invoke(index.clj:2146)
	at datomic.index$merge_db.invokeStatic(index.clj:2156)
	at datomic.index$merge_db$fn__13940.invoke(index.clj:2165)
	at datomic.index$merge_db$fn__13940$fn__13944.invoke(index.clj:2172)
	at datomic.index$merge_db_STAR_.invoke(index.clj:1954)
	at datomic.index$merge_db_STAR_.invokeStatic(index.clj:2067)
	at datomic.index$merge_db_STAR_$build_tiered_index__13838.invoke(index.clj:2043)
	at datomic.index$merge_db_STAR_$build_tiered_index__13838$fn__13888.invoke(index.clj:2050)
	at datomic.index$merge_db_STAR_$build_tiered_index__13838$build_index__13842.invoke(index.clj:1972)
	at datomic.index$merge_one_index.invoke(index.clj:1404)
	at datomic.index$merge_one_index.invokeStatic(index.clj:1468)
	at datomic.index$merge_one_index$f__13377__auto____13546.invoke(index.clj:1508)
	at datomic.index$bounded_count.invoke(index.clj:964)
	at datomic.index$bounded_count.invokeStatic(index.clj:967)
	at clojure.core$next__5451.invoke(core.clj:64)
	at clojure.core$next__5451.invokeStatic(core.clj:64)
	at clojure.lang.RT.next(RT.java:713)
	at clojure.lang.Cons.next(Cons.java:39)
	at clojure.lang.LazySeq.seq(LazySeq.java:51)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at datomic.index$fully_take_while_delivering_tail$fn__13433.invoke(index.clj:1048)
	at clojure.core$seq__5467.invoke(core.clj:139)
	at clojure.core$seq__5467.invokeStatic(core.clj:139)
	at clojure.lang.RT.seq(RT.java:535)
	at clojure.lang.LazySeq.seq(LazySeq.java:51)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at clojure.core$filter$fn__5962.invoke(core.clj:2826)
	at clojure.core$seq__5467.invokeStatic(core.clj:139)
	at clojure.lang.RT.seq(RT.java:535)
	at clojure.lang.LazySeq.seq(LazySeq.java:51)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at clojure.core$filter$fn__5962.invoke(core.clj:2826)
	at clojure.core$seq__5467.invokeStatic(core.clj:139)
	at clojure.lang.RT.seq(RT.java:535)
	at clojure.lang.LazySeq.seq(LazySeq.java:51)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at clojure.core$concat$cat__5560$fn__5561.invoke(core.clj:744)
	at clojure.core$next__5451.invokeStatic(core.clj:64)
	at clojure.lang.RT.next(RT.java:713)
	at clojure.lang.Cons.next(Cons.java:39)
	at clojure.lang.LazySeq.seq(LazySeq.java:51)
	at clojure.lang.LazySeq.sval(LazySeq.java:42)
	at clojure.core$map$fn__5935.invoke(core.clj:2772)
	at datomic.index$merge_db_STAR_$build_tiered_index__13838$fn__13888$fn__13898.invoke(index.clj:2050)
	at datomic.cache$getx_uncached.invoke(cache.clj:57)
	at datomic.cache$getx_uncached.invokeStatic(cache.clj:59)
	at datomic.cache$lookup_cache$reify__11891.valAtUncached(cache.clj:169)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.cache$lookup_with_inflight_cache$reify__11894.valAt(cache.clj:187)
	at clojure.core$deref.invoke(core.clj:2323)
	at clojure.core$deref.invokeStatic(core.clj:2337)
	at clojure.lang.Delay.deref(Delay.java:42)
	at datomic.cache$lookup_with_inflight_cache$reify__11894$fn__11895.invoke(cache.clj:184)
	at clojure.lang.RT.get(RT.java:791)
	at datomic.domain.DeserializingRepairingLookup.valAt(domain.clj:214)
	at datomic.domain.DeserializingRepairingLookup$fn__17950.invoke(domain.clj:217)
	at datomic.cache$report_val_fn_fail.invoke(cache.clj:65)
	at datomic.cache$report_val_fn_fail.invokeStatic(cache.clj:69)