Convert .dat recordings into individual JSON files - new menu item added in PrimaryView.fxml

ulyp-storage/build.gradle

@@ -23,6 +23,7 @@ dependencies {
     implementation group: 'org.agrona', name: 'agrona', version: '1.4.0'
     implementation group: 'org.jetbrains', name: 'annotations', version: '18.0.0'
     implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.2'
+    implementation group: 'it.unimi.dsi', name: 'fastutil', version: '8.5.12'
 
     compileOnly group: 'org.rocksdb', name: 'rocksdbjni', version: '9.7.3'
     testImplementation group: 'org.rocksdb', name: 'rocksdbjni', version: '9.7.3'

ulyp-ui/src/main/kotlin/com/ulyp/ui/PrimaryView.kt

@@ -11,10 +11,10 @@ import com.ulyp.ui.elements.controls.ErrorModalView
 import com.ulyp.ui.elements.misc.ExceptionAsTextView
 import com.ulyp.ui.elements.recording.tree.FileRecordingTabPane
 import com.ulyp.ui.elements.recording.tree.FileRecordingsTabName
+import com.ulyp.ui.export.RecordingMultipleJsonExporter
 import com.ulyp.ui.reader.ReaderRegistry
 import com.ulyp.ui.settings.Settings
 import com.ulyp.ui.util.FxThreadExecutor
-import com.ulyp.ui.export.RecordingJsonExporter
 import javafx.application.Platform
 import javafx.fxml.FXML
 import javafx.fxml.FXMLLoader
@@ -31,6 +31,7 @@ import org.springframework.context.ApplicationContext
 import java.io.File
 import java.net.URL
 import java.util.*
+import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Supplier
 import kotlin.system.exitProcess
 
@@ -133,26 +134,39 @@ class PrimaryView(
         popup.show()
     }
 
-    /**
-     * Exports currently recordings to JSON files to same directory where recording file resides
-     */
-    fun exportToJson() {
+    fun openRecordingFile() {
         val file: File = fileChooser.get() ?: return
         val callRecordTree: CallRecordTree = readerRegistry.newCallRecordTree(file) ?: return
 
+        val fileRecordingsTab = fileRecordingTabPane.getOrCreateProcessTab(
+            FileRecordingsTabName(file, callRecordTree.processMetadata)
+        )
+        fileRecordingsTab.setOnClosed {
+            readerRegistry.dispose(callRecordTree)
+            callRecordTree.close()
+        }
+
+        val rocksdbAvailable = RocksdbChecker.checkRocksdbAvailable()
+        if (!rocksdbAvailable.value()) {
+            val errorPopup = applicationContext.getBean(
+                ErrorModalView::class.java,
+                applicationContext.getBean(SceneRegistry::class.java),
+                "Rocksdb is not available on your platform, in-memory index will be used. Please note this may cause OOM on large recordings",
+                ExceptionAsTextView(rocksdbAvailable.err!!)
+            )
+            errorPopup.show()
+        }
+
         val loadingProgressBar = ProgressBar()
         loadingProgressBar.prefWidth = 200.0
         fileTabPaneAnchorPane.children.add(loadingProgressBar)
 
-        val recordings = mutableMapOf<Int, Recording>()
-
-        // Collect all recordings from the file (latest states)
         callRecordTree.subscribe(
             object : RecordingListener {
                 var prevProgress = 0.0
 
                 override fun onRecordingUpdated(recording: Recording) {
-                    recordings[recording.id] = recording
+                    fileRecordingsTab.updateOrCreateRecordingTab(callRecordTree.processMetadata, recording)
                 }
 
                 override fun onProgressUpdated(progress: Double) {
@@ -170,10 +184,10 @@ class PrimaryView(
         callRecordTree.completeFuture.exceptionally {
             FxThreadExecutor.execute {
                 val errorPopup = applicationContext.getBean(
-                    ErrorModalView::class.java,
-                    applicationContext.getBean(SceneRegistry::class.java),
-                    "Stopped reading recording file $file with error: " + it.message,
-                    ExceptionAsTextView(it)
+                        ErrorModalView::class.java,
+                        applicationContext.getBean(SceneRegistry::class.java),
+                        "Stopped reading recording file $file with error: " + it.message,
+                        ExceptionAsTextView(it)
                 )
                 errorPopup.show()
             }
@@ -183,25 +197,15 @@ class PrimaryView(
                 loadingProgressBar.visibleProperty().set(false)
                 fileTabPaneAnchorPane.children.remove(loadingProgressBar)
             }
-            recordings.forEach { _, recording ->
-                val outDir = file.parentFile ?: File(".")
-                val outFile = File(outDir, "${file.name}-recording-${recording.id}.json")
-                RecordingJsonExporter.export(recording, outFile)
-            }
         }
     }
 
-    fun openRecordingFile() {
+    // Exports recordings from a selected file to JSON format
+    fun exportToJson() {
         val file: File = fileChooser.get() ?: return
-        val callRecordTree: CallRecordTree = readerRegistry.newCallRecordTree(file) ?: return
 
-        val fileRecordingsTab = fileRecordingTabPane.getOrCreateProcessTab(
-            FileRecordingsTabName(file, callRecordTree.processMetadata)
-        )
-        fileRecordingsTab.setOnClosed {
-            readerRegistry.dispose(callRecordTree)
-            callRecordTree.close()
-        }
+        // Create a new CallRecordTree to read the file in the background (same as openRecordingFile)
+        val callRecordTree: CallRecordTree = readerRegistry.newCallRecordTree(file) ?: return
 
         val rocksdbAvailable = RocksdbChecker.checkRocksdbAvailable()
         if (!rocksdbAvailable.value()) {
@@ -214,16 +218,20 @@ class PrimaryView(
             errorPopup.show()
         }
 
-        val loadingProgressBar = ProgressBar()
-        loadingProgressBar.prefWidth = 200.0
+        val loadingProgressBar = ProgressBar().apply {
+            prefWidth = 200.0
+        }
         fileTabPaneAnchorPane.children.add(loadingProgressBar)
 
+        // onRecordingUpdated can be called multiple times per recording; capture the latest snapshot per id
+        val recordingsById = ConcurrentHashMap<Int, Recording>()
+
         callRecordTree.subscribe(
             object : RecordingListener {
                 var prevProgress = 0.0
 
                 override fun onRecordingUpdated(recording: Recording) {
-                    fileRecordingsTab.updateOrCreateRecordingTab(callRecordTree.processMetadata, recording)
+                    recordingsById[recording.id] = recording
                 }
 
                 override fun onProgressUpdated(progress: Double) {
@@ -238,22 +246,53 @@ class PrimaryView(
             }
         )
 
-        callRecordTree.completeFuture.exceptionally {
-            FxThreadExecutor.execute {
-                val errorPopup = applicationContext.getBean(
+        callRecordTree.completeFuture.whenComplete { _, err ->
+            try {
+                if (err != null) {
+                    FxThreadExecutor.execute {
+                        val errorPopup = applicationContext.getBean(
+                            ErrorModalView::class.java,
+                            applicationContext.getBean(SceneRegistry::class.java),
+                            "Stopped reading recording file $file with error: " + err.message,
+                            ExceptionAsTextView(err)
+                        )
+                        errorPopup.show()
+                    }
+                    return@whenComplete
+                }
+
+                val outDir = file.parentFile ?: File(".")
+                recordingsById.values
+                    .sortedBy { it.id }
+                    .forEach { recording ->
+                        val outFile = File(outDir, "recording-${recording.id}.json")
+                        println("Exporting recording ${recording.id} to JSON file: ${outFile.absolutePath}")
+                        RecordingMultipleJsonExporter.export(recording, outFile)
+                    }
+
+            } catch (e: Exception) {
+                FxThreadExecutor.execute {
+                    val errorPopup = applicationContext.getBean(
                         ErrorModalView::class.java,
                         applicationContext.getBean(SceneRegistry::class.java),
-                        "Stopped reading recording file $file with error: " + it.message,
-                        ExceptionAsTextView(it)
-                )
-                errorPopup.show()
-            }
-            null
-        }.thenAccept {
-            FxThreadExecutor.execute {
-                loadingProgressBar.visibleProperty().set(false)
-                fileTabPaneAnchorPane.children.remove(loadingProgressBar)
+                        "Failed to export $file to JSON: " + e.message,
+                        ExceptionAsTextView(e)
+                    )
+                    errorPopup.show()
+                }
+            } finally {
+                FxThreadExecutor.execute {
+                    loadingProgressBar.visibleProperty().set(false)
+                    fileTabPaneAnchorPane.children.remove(loadingProgressBar)
+                }
+                try {
+                    readerRegistry.dispose(callRecordTree)
+                    callRecordTree.close()
+                } catch (_: Exception) {
+                    // best effort cleanup
+                }
             }
         }
     }
+
 }

ulyp-ui/src/main/kotlin/com/ulyp/ui/export/AllRecordingsJson.kt

@@ -0,0 +1,11 @@
+package com.ulyp.ui.export
+
+/**
+ * The "all recordings in one file" JSON.
+ * - count: how many recordings we exported
+ * - recordings: left-pane summary + right-pane tree per recording
+ */
+data class AllRecordingsJson(
+    val count: Int,
+    val recordings: List<RecordingJson>
+)

ulyp-ui/src/main/kotlin/com/ulyp/ui/export/RecordingJsonConverter.kt

@@ -0,0 +1,141 @@
+package com.ulyp.ui.export
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
+import com.ulyp.core.recorders.ObjectRecord
+import com.ulyp.storage.tree.CallRecord
+import com.ulyp.storage.tree.CallRecordTree
+import com.ulyp.storage.tree.Recording
+import java.io.File
+
+/**
+ * Exports a parsed Recording (Java object) to JSON shaped like your UI.
+ *
+ * LEFT PANE:
+ *  - id, threadName, startTimeEpochMs, durationMillis, totalCalls (all from Recording/RecordingMetadata)
+ *
+ * RIGHT PANE:
+ *  - root call (CallRecord) recursively expanded to children:
+ *      ownerClass.methodName(args) -> returnValue / thrown / durationNanos
+ *
+ */
+object RecordingJsonConverter {
+
+    // Single ObjectMapper instance; pretty printing enabled for readability in files.
+    private val mapper = ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)
+
+    /**
+    * Export ALL currently published recordings in a CallRecordTree to ONE JSON file.
+    *
+    * Safe to call multiple times while parsing progresses: it just overwrites the file
+    * with the latest list from tree.getRecordings().
+    */
+    fun exportAll(tree: CallRecordTree, outFile: File) {
+        // Pull all *published* recordings. CallRecordTree already filters via RecordingState.isPublished().
+        val recordings: List<Recording> = tree.recordings
+
+        // Convert each to the same JSON shape you already use for single-file exports.
+        val list: List<RecordingJson> = recordings.map(::toRecordingJson)
+
+        // Wrap into a single payload
+        val payload = AllRecordingsJson(
+            count = list.size,
+            recordings = list
+        )
+
+        outFile.parentFile?.mkdirs()
+        mapper.writeValue(outFile, payload)
+        println("Exported ${list.size} recordings to ${outFile.absolutePath}")
+    }
+
+    // -------------- Internal: per-recording conversion (left summary + right tree) --------------
+
+    /** Builds RecordingJson (left-pane summary + right-pane tree) from a Recording. */
+    fun toRecordingJson(recording: Recording): RecordingJson {
+        // LEFT PANE: all precomputed in your model
+        // Thread name / start time come from RecordingMetadata
+        val md = recording.metadata
+        // Recording identity
+        val id: Int = recording.id
+        val threadName: String? = md.threadName
+        val startEpochMs: Long? = md.recordingStartedMillis
+        // Root call duration (Duration) -> millis (matches UI "ms")
+        val durationMs: Long = recording.rootDuration().toMillis()
+        // Total number of calls in this recording (matches UI count)
+        val totalCalls: Int = recording.callCount()
+
+
+        // RIGHT PANE: walk the CallRecord tree starting from the root
+
+        // Your Recording always has exactly one root call
+        val rootCall: CallRecord = recording.root
+        // Recursively convert the root CallRecord to a DTO tree
+        val rootDto: NodeJson = toNodeJson(rootCall)
+
+        // -------- Build the top-level DTO --------
+        val recordingJson = RecordingJson(
+            id = id,
+            threadName = threadName,
+            startTimeEpochMs = startEpochMs,
+            durationMillis = durationMs,
+            totalCalls = totalCalls,
+            root = rootDto
+        )
+
+        println("Converted recording $id with $totalCalls calls to JSON DTO")
+        return recordingJson
+    }
+
+
+    /**
+     * RECURSION:
+     *  Converts a CallRecord (one node) into NodeJson, then does the same for each child.
+     */
+    private fun toNodeJson(node: CallRecord): NodeJson {
+        // Stable node id within the recording session (starts at 0 per your docs)
+        val nodeId: Long? = node.id
+
+        // Method owner (class) and method name come from node.getMethod()
+        val ownerClass: String? = node.method.type.name
+        val methodName: String? = node.method.name
+
+        // Arguments and return value are ObjectRecord; stringifying is enough for the JSON
+        val args: List<String> = node.args.map(::renderObjectRecord)
+        val returnValue: String? = renderObjectRecordOrNull(node.returnValue)
+
+        // Whether the method threw during this call
+        val thrown: Boolean = node.hasThrown()
+
+        // Duration of this call in nanoseconds (per-node timing available in CallRecord)
+        val durationNanos: Long = node.nanosDuration
+
+        // Children: CallRecord resolves children lazily via RecordingState using child ids
+        val children: List<NodeJson> = node.getChildren().map { child -> toNodeJson(child) }
+
+        return NodeJson(
+            nodeId = nodeId,
+            ownerClass = ownerClass,
+            methodName = methodName,
+            args = args,
+            returnValue = returnValue,
+            thrown = thrown,
+            durationNanos = durationNanos,
+            children = children
+        )
+    }
+
+    // ---------- Small helpers to render ObjectRecord safely ----------
+
+    private fun renderObjectRecord(obj: ObjectRecord): String {
+        // ObjectRecord implementations have sensible toString() for UI display.
+        // Using toString() keeps JSON aligned with what the right pane shows.
+        return obj.toString()
+    }
+
+    private fun renderObjectRecordOrNull(obj: ObjectRecord?): String? {
+        // Some return values may be "not recorded"; toString() still works,
+        // but we'll keep it nullable in case an implementation returns null.
+        return obj?.toString()
+    }
+
+}

ulyp-ui/src/main/kotlin/com/ulyp/ui/export/RecordingMultipleJsonExporter.kt

@@ -0,0 +1,25 @@
+package com.ulyp.ui.export
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.SerializationFeature
+import com.ulyp.storage.tree.Recording
+import java.io.File
+
+object RecordingMultipleJsonExporter {
+
+    // Single ObjectMapper instance; pretty printing enabled for readability in files.
+    private val mapper = ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)
+
+    /**
+     * Export one Recording to a JSON file on disk.
+     *
+     * @param recording Parsed Recording object (NOT raw bytes)
+     * @param outFile   Destination JSON file path
+     */
+    fun export(recording: Recording, outFile: File) {
+        val dto = RecordingJsonConverter.toRecordingJson(recording)
+        outFile.parentFile?.mkdirs()
+        mapper.writeValue(outFile, dto)
+    }
+
+}