From e1419dc1c6a2e61d8f9ed3988ba782f83fa45fbd Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Thu, 11 Sep 2025 08:04:10 -0700 Subject: [PATCH 01/14] Initial commit, get event info for system and regions --- PRASReport.jl/LICENSE.md | 21 ++++++ PRASReport.jl/Project.toml | 30 ++++++++ PRASReport.jl/README.md | 0 PRASReport.jl/TODOs.md | 12 ++++ PRASReport.jl/examples/run_report.jl | 16 +++++ PRASReport.jl/src/PRASReport.jl | 23 ++++++ PRASReport.jl/src/events.jl | 104 +++++++++++++++++++++++++++ 7 files changed, 206 insertions(+) create mode 100644 PRASReport.jl/LICENSE.md create mode 100644 PRASReport.jl/Project.toml create mode 100644 PRASReport.jl/README.md create mode 100644 PRASReport.jl/TODOs.md create mode 100644 PRASReport.jl/examples/run_report.jl create mode 100644 PRASReport.jl/src/PRASReport.jl create mode 100644 PRASReport.jl/src/events.jl diff --git a/PRASReport.jl/LICENSE.md b/PRASReport.jl/LICENSE.md new file mode 100644 index 00000000..fc56637f --- /dev/null +++ b/PRASReport.jl/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Alliance for Sustainable Energy, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PRASReport.jl/Project.toml b/PRASReport.jl/Project.toml new file mode 100644 index 00000000..f4372ff7 --- /dev/null +++ b/PRASReport.jl/Project.toml @@ -0,0 +1,30 @@ +name = "PRASReport" +uuid = "c003f3f0-f5d5-4077-8f93-207e59ecb3ff" +authors = ["Hari Sundar "] +version = "0.1.0" + +[deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +DuckDB = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +PRAS = "05348d26-1c52-11e9-35e3-9d51842d34b9" +PRASCapacityCredits = "2e1a2ed5-e89d-4cd3-bc86-c0e88a73d3a3" +PRASCore = "c5c32b99-e7c3-4530-a685-6f76e19f7fe2" +PRASFiles = "a2806276-6d43-4ef5-91c0-491704cd7cf1" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" + +[compat] +Dates = "1" +JSON3 = "1.14" +PRASCore = "0.8.0" +StatsBase = "0.34" +TimeZones = "^1.14" +julia = "1.10" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/PRASReport.jl/README.md b/PRASReport.jl/README.md new file mode 100644 index 00000000..e69de29b diff --git a/PRASReport.jl/TODOs.md b/PRASReport.jl/TODOs.md new file mode 100644 index 00000000..eeab5c16 --- /dev/null +++ b/PRASReport.jl/TODOs.md @@ -0,0 +1,12 @@ +# TODO +0. Store DB as base64 +1. Landing page shows most important characteristics like + - Number of events with total EUE exceeds the ## number + - Same with LOLE + - Top 1 ptile events + - Event summary graph (like the ESIG report) +2. Users from the webpage can specify which regions need to be aggregated to get combined metric +3. PRASReport exports an assess function which automatically runs assess with Shortfall(), Surplus(), Flows(), and Utilization() +4. Event selector creates a tab for an event to show regional shortfall and flows etc +5. Does NEUE have to be reported as MWh/MWh +6. Explore how a directed graph can be stored in DuckDB, and how it can be drawn on a webpage with WASM-DuckDB \ No newline at end of file diff --git a/PRASReport.jl/examples/run_report.jl b/PRASReport.jl/examples/run_report.jl new file mode 100644 index 00000000..8976aa0e --- /dev/null +++ b/PRASReport.jl/examples/run_report.jl @@ -0,0 +1,16 @@ +using PRAS +using PRASReport + +rts_sys = rts_gmlc(); +rts_sys.regions.load .+= 375; + +sf, = assess(rts_sys,SequentialMonteCarlo(samples=100),Shortfall()); + +event_threshold = 0 +events = get_events(sf,event_threshold) + +println("System $(EUE(sf))") +println("System $(NEUE(sf))") +println("Number of events where each event-hour in the event has EUE > $event_threshold MW: ", length(events)) +println("Longest event is over a period of ", maximum(length.(events))) + diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl new file mode 100644 index 00000000..229e2403 --- /dev/null +++ b/PRASReport.jl/src/PRASReport.jl @@ -0,0 +1,23 @@ +module PRASReport + +import PRASCore.Systems: SystemModel, Regions, Interfaces, + Generators, Storages, GeneratorStorages, Lines, + timeunits, powerunits, energyunits, unitsymbol + +import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, + ShortfallSamplesResult, AbstractShortfallResult, + Result, MeanEstimate, findfirstunique, + val, stderror +import StatsBase: mean +import Dates: @dateformat_str, format, now +import TimeZones: ZonedDateTime + +using DuckDB +using Base64 +using JSON3 + +include("events.jl") + +export Event, get_events + +end # module PRASReport diff --git a/PRASReport.jl/src/events.jl b/PRASReport.jl/src/events.jl new file mode 100644 index 00000000..04c32aae --- /dev/null +++ b/PRASReport.jl/src/events.jl @@ -0,0 +1,104 @@ +mutable struct Event{N,L,T,E} + name::String + timestamps::StepRange{ZonedDateTime,T} + lole::Vector{LOLE} + eue::Vector{EUE} + neue::Vector{NEUE} + regions::Vector{String} + + function Event{}(name::String, timestamps::StepRange{ZonedDateTime,T}, lole::Vector{LOLE{N,L,T}}, eue::Vector{EUE{N,L,T,E}}, neue::Vector{NEUE}, regions::Vector{String}) where {N,L,T,E} + @assert length(lole) == length(eue) == length(neue) "Length of lole, eue, and neue vectors must be equal" + new{N,L,T,E}(name, timestamps, lole, eue, neue, regions) + end +end + +Base.length(event::Event{N,L,T}) where {N,L,T} = T(N*L) + +function get_events(sf::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} + + eue_system = EUE.(sf,sf.timestamps) + system_eue_above_threshold = findall(val.(eue_system) .> event_threshold) + event_timegroups = get_stepranges(sf.timestamps[system_eue_above_threshold],L,T) + + return map(ts_group -> Event(sf,ts_group, + format(first(ts_group),"yyyy-mm-dd HH:MM ZZZ") + ), + event_timegroups) + +end + +function Event(sf::ShortfallResult{N,L,T,E}, + event_timestamps::StepRange{ZonedDateTime,T}, + name::String=nothing + ) where {N,L,T,E} + + if isnothing(name) + name = "Shortfall Event" + end + event_length = length(event_timestamps) + ts_first = findfirstunique(sf.timestamps,first(event_timestamps)) + ts_last = findfirstunique(sf.timestamps,last(event_timestamps)) + + lole = Vector{LOLE{event_length,L,T}}() + eue = Vector{EUE{event_length,L,T,E}}() + neue = Vector{NEUE}() + push!(lole, + LOLE{event_length,L,T}( + MeanEstimate(sum(val.(LOLE.(sf,event_timestamps)))) + ) + ) + push!(eue, + EUE{event_length,L,T,E}( + MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))) + ) + ) + + push!(neue, + NEUE( + div(MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))), + sum(sf.regions.load[:,ts_first:ts_last])/1e6)) + ) + + for (r,region) in enumerate(sf.regions.names) + + push!(lole, + LOLE{event_length,L,T}( + MeanEstimate(sum(val.(LOLE.(sf,region,event_timestamps)))) + ) + ) + push!(eue, + EUE{event_length,L,T,E}( + MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))) + ) + ) + + push!(neue, + NEUE( + div(MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))), + sum(sf.regions.load[r,ts_first:ts_last])/1e6)) + ) + end + + # TODO: Change variable name + # TODO: Should all events have common region_names? + region_names = ["System",sf.regions.names...] + + return Event(name,event_timestamps,lole,eue,neue,region_names) +end + +function get_stepranges(vec::Vector{ZonedDateTime},L,T) + groups = Vector{StepRange{ZonedDateTime,T}}() + start = vec[1] + final = vec[1] + for next in vec[2:end] + if next == final + T(L) + final = next + else + push!(groups, StepRange(start,T(L),final)) + start = next + final = next + end + end + push!(groups, StepRange(start,T(L),final)) + return groups +end \ No newline at end of file From cc7ecc94fef119d76aca50c0853e1761f220e22b Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Tue, 16 Sep 2025 13:34:51 -0600 Subject: [PATCH 02/14] Make changes for copperplate system, more checks when creating events, start tests --- PRASReport.jl/TODOs.md | 5 +- PRASReport.jl/examples/run_report.jl | 2 +- PRASReport.jl/src/PRASReport.jl | 2 +- PRASReport.jl/src/events.jl | 128 +++++++++++++++++---------- PRASReport.jl/test/runtests.jl | 42 +++++++++ 5 files changed, 126 insertions(+), 53 deletions(-) create mode 100644 PRASReport.jl/test/runtests.jl diff --git a/PRASReport.jl/TODOs.md b/PRASReport.jl/TODOs.md index eeab5c16..8cf068f7 100644 --- a/PRASReport.jl/TODOs.md +++ b/PRASReport.jl/TODOs.md @@ -1,5 +1,5 @@ # TODO -0. Store DB as base64 +0. Store DB as base64, or data URL in the html 1. Landing page shows most important characteristics like - Number of events with total EUE exceeds the ## number - Same with LOLE @@ -9,4 +9,5 @@ 3. PRASReport exports an assess function which automatically runs assess with Shortfall(), Surplus(), Flows(), and Utilization() 4. Event selector creates a tab for an event to show regional shortfall and flows etc 5. Does NEUE have to be reported as MWh/MWh -6. Explore how a directed graph can be stored in DuckDB, and how it can be drawn on a webpage with WASM-DuckDB \ No newline at end of file +6. Explore how a directed graph can be stored in DuckDB, and how it can be drawn on a webpage with WASM-DuckDB +7. Julia indentation \ No newline at end of file diff --git a/PRASReport.jl/examples/run_report.jl b/PRASReport.jl/examples/run_report.jl index 8976aa0e..2d094777 100644 --- a/PRASReport.jl/examples/run_report.jl +++ b/PRASReport.jl/examples/run_report.jl @@ -12,5 +12,5 @@ events = get_events(sf,event_threshold) println("System $(EUE(sf))") println("System $(NEUE(sf))") println("Number of events where each event-hour in the event has EUE > $event_threshold MW: ", length(events)) -println("Longest event is over a period of ", maximum(length.(events))) +println("Longest event is over a period of ", maximum(event_length.(events))) diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl index 229e2403..685979b7 100644 --- a/PRASReport.jl/src/PRASReport.jl +++ b/PRASReport.jl/src/PRASReport.jl @@ -18,6 +18,6 @@ using JSON3 include("events.jl") -export Event, get_events +export Event, get_events, event_length end # module PRASReport diff --git a/PRASReport.jl/src/events.jl b/PRASReport.jl/src/events.jl index 04c32aae..5a3f231b 100644 --- a/PRASReport.jl/src/events.jl +++ b/PRASReport.jl/src/events.jl @@ -6,26 +6,26 @@ mutable struct Event{N,L,T,E} neue::Vector{NEUE} regions::Vector{String} - function Event{}(name::String, timestamps::StepRange{ZonedDateTime,T}, lole::Vector{LOLE{N,L,T}}, eue::Vector{EUE{N,L,T,E}}, neue::Vector{NEUE}, regions::Vector{String}) where {N,L,T,E} - @assert length(lole) == length(eue) == length(neue) "Length of lole, eue, and neue vectors must be equal" + function Event{}(name::String, timestamps::StepRange{ZonedDateTime,T}, + lole::Vector{LOLE{N,L,T}}, eue::Vector{EUE{N,L,T,E}}, + neue::Vector{NEUE}, regions::Vector{String} + ) where {N,L,T,E} + + length(lole) != length(eue) != length(neue) != length(regions) && + error("Length of lole, eue, neue, and region names vectors must be equal") + + length(timestamps) != N && + error("Number of timesteps should match metrics event length") + + length(regions) > 1 && !isapprox(val(eue[1]),sum(val.(eue[2:end]))) && + error("First value in an event eue array should represent system EUE" + *"which is approximately the sum of EUE of all the regions") + new{N,L,T,E}(name, timestamps, lole, eue, neue, regions) end end -Base.length(event::Event{N,L,T}) where {N,L,T} = T(N*L) - -function get_events(sf::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} - - eue_system = EUE.(sf,sf.timestamps) - system_eue_above_threshold = findall(val.(eue_system) .> event_threshold) - event_timegroups = get_stepranges(sf.timestamps[system_eue_above_threshold],L,T) - - return map(ts_group -> Event(sf,ts_group, - format(first(ts_group),"yyyy-mm-dd HH:MM ZZZ") - ), - event_timegroups) - -end +event_length(event::Event{N,L,T}) where {N,L,T} = T(N*L) function Event(sf::ShortfallResult{N,L,T,E}, event_timestamps::StepRange{ZonedDateTime,T}, @@ -35,57 +35,87 @@ function Event(sf::ShortfallResult{N,L,T,E}, if isnothing(name) name = "Shortfall Event" end + event_length = length(event_timestamps) ts_first = findfirstunique(sf.timestamps,first(event_timestamps)) ts_last = findfirstunique(sf.timestamps,last(event_timestamps)) - lole = Vector{LOLE{event_length,L,T}}() - eue = Vector{EUE{event_length,L,T,E}}() - neue = Vector{NEUE}() - push!(lole, - LOLE{event_length,L,T}( + + + lole = [LOLE{event_length,L,T}( MeanEstimate(sum(val.(LOLE.(sf,event_timestamps)))) - ) - ) - push!(eue, - EUE{event_length,L,T,E}( + )] + + eue = [EUE{event_length,L,T,E}( MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))) - ) - ) + )] - push!(neue, - NEUE( + neue = [NEUE( div(MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))), - sum(sf.regions.load[:,ts_first:ts_last])/1e6)) - ) + sum(sf.regions.load[:,ts_first:ts_last])/1e6))] - for (r,region) in enumerate(sf.regions.names) + if length(sf.regions) == 1 + # TODO: Change variable name + # TODO: Should all events have common region_names? + region_names = sf.regions.names + else + region_names = ["System"] - push!(lole, - LOLE{event_length,L,T}( - MeanEstimate(sum(val.(LOLE.(sf,region,event_timestamps)))) - ) - ) - push!(eue, - EUE{event_length,L,T,E}( - MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))) + for (r,region) in enumerate(sf.regions.names) + + push!(lole, + LOLE{event_length,L,T}( + MeanEstimate(sum(val.(LOLE.(sf,region,event_timestamps)))) ) ) + push!(eue, + EUE{event_length,L,T,E}( + MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))) + ) + ) - push!(neue, - NEUE( - div(MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))), - sum(sf.regions.load[r,ts_first:ts_last])/1e6)) - ) + push!(neue, + NEUE( + div(MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))), + sum(sf.regions.load[r,ts_first:ts_last])/1e6)) + ) + + push!(region_names,region) + end end - # TODO: Change variable name - # TODO: Should all events have common region_names? - region_names = ["System",sf.regions.names...] - return Event(name,event_timestamps,lole,eue,neue,region_names) end +""" + get_events(sf::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} + +Extracts events from PRAS ShortfallResult objects where an event is a contiguous +period during which the system EUE exceeds a specified threshold, and +returns a vector of (@ref Event) objects. + +If the PRAS simulation is hourly and event_threshold is 0, and there are +5 consecutive hours where the system EUE exceeds the threshold, this returns a +vector with a single event. +""" +function get_events(sf::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} + + event_threshold < 0 && error("Event threshold must be non-negative") + + eue_system = EUE.(sf,sf.timestamps) + system_eue_above_threshold = findall(val.(eue_system) .> event_threshold) + + isempty(system_eue_above_threshold) && error("No shortfall events in this simulation") + + event_timegroups = get_stepranges(sf.timestamps[system_eue_above_threshold],L,T) + + return map(ts_group -> Event(sf,ts_group, + format(first(ts_group),"yyyy-mm-dd HH:MM ZZZ") + ), + event_timegroups) + +end + function get_stepranges(vec::Vector{ZonedDateTime},L,T) groups = Vector{StepRange{ZonedDateTime,T}}() start = vec[1] diff --git a/PRASReport.jl/test/runtests.jl b/PRASReport.jl/test/runtests.jl new file mode 100644 index 00000000..69afac67 --- /dev/null +++ b/PRASReport.jl/test/runtests.jl @@ -0,0 +1,42 @@ +using Test +using PRASReport +using TimeZones +using Dates + + +@testset "PRASReport.jl Tests" begin + @testset "Test events" begin + + eue = [EUE{2,1,Hour,MWh}(MeanEstimate(1.2)), + EUE{2,1,Hour,MWh}(MeanEstimate(0.6)), + EUE{2,1,Hour,MWh}(MeanEstimate(0.5)), + EUE{2,1,Hour,MWh}(MeanEstimate(0.1)), + ] + + lole = [LOLE{4380,2,Hour}(MeanEstimate(0.13)), + LOLE{4380,2,Hour}(MeanEstimate(0.08)), + LOLE{4380,2,Hour}(MeanEstimate(0.06)), + LOLE{4380,2,Hour}(MeanEstimate(0.04)) + + ] + + @test event_length(event) == Hour(5) + end + + @testset "Test get_events" begin + + #TODO: Test for empty sf object + + end + @testset "Test get_stepranges" begin + + timestamps = ZonedDateTime.(DateTime(2023,1,1):Hour(1):DateTime(2023,1,10), tz"UTC") + selected_times = [timestamps[2], timestamps[3], timestamps[5], timestamps[6], timestamps[7], timestamps[9]] + step_ranges = get_stepranges(selected_times, Hour(1), Hour(1)) + + @test length(step_ranges) == 3 "Incorrect number of step ranges" + @test step_ranges[1] == (timestamps[2]:Hour(1):timestamps[3]) "First step range incorrect" + @test step_ranges[2] == (timestamps[5]:Hour(1):timestamps[7]) "Second step range incorrect" + @test step_ranges[3] == (timestamps[9]:Hour(1):timestamps[9]) "Third step range incorrect" + end +end From f9f060e96e896316ac3e427a5cfcac54a62a38cf Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Fri, 19 Sep 2025 12:41:43 -0600 Subject: [PATCH 03/14] Add sf and flow structs --- PRASReport.jl/src/PRASReport.jl | 4 +++- PRASReport.jl/src/events.jl | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl index 685979b7..b9bd821e 100644 --- a/PRASReport.jl/src/PRASReport.jl +++ b/PRASReport.jl/src/PRASReport.jl @@ -4,7 +4,9 @@ import PRASCore.Systems: SystemModel, Regions, Interfaces, Generators, Storages, GeneratorStorages, Lines, timeunits, powerunits, energyunits, unitsymbol -import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, +import PRASCore.Simulations: assess + +import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, FlowResult, ShortfallSamplesResult, AbstractShortfallResult, Result, MeanEstimate, findfirstunique, val, stderror diff --git a/PRASReport.jl/src/events.jl b/PRASReport.jl/src/events.jl index 5a3f231b..da2db82b 100644 --- a/PRASReport.jl/src/events.jl +++ b/PRASReport.jl/src/events.jl @@ -25,6 +25,36 @@ mutable struct Event{N,L,T,E} end end +mutable struct sf_ts{} + name::String + timestamps::Vector{ZonedDateTime} + eue::Vector{Vector{EUE}} + lole::Vector{Vector{LOLE}} + neue::Vector{Vector{NEUE}} + regions::Vector{String} + + function sf_ts(event,sf{N,L,T,E}) where {N,L,T,E} + name = event.name + timestamps = collect(event.timestamps) + eue = EUE{1,L,T,E}(MeanEstimate(val.(EUE.(sf,:,timestamps)))) + lole = LOLE{1,L,T}(MeanEstimate(val.(LOLE.(sf,:,timestamps)))) + neue = NEUE(MeanEstimate(val.(NEUE.(sf,:,timestamps)))) + regions = event.regions[2:end] + new(name,timestamps,[eue],[lole],[neue],regions) + end +end + +mutable struct flow_ts{} + name::String + timestamps::Vector{ZonedDateTime} + flow::Vector{Vector{NEUE}} + interfaces::Vector{String} + function flow_ts(timestamps::Vector{ZonedDateTime}) + new(timestamps) + end +end + + event_length(event::Event{N,L,T}) where {N,L,T} = T(N*L) function Event(sf::ShortfallResult{N,L,T,E}, From f466c16d7335debe6f07cdeaf9789475d50c57db Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Fri, 19 Sep 2025 22:58:46 -0600 Subject: [PATCH 04/14] Create SQL db schema, write functions and getting db etc --- PRASCore.jl/src/Results/Results.jl | 3 + PRASCore.jl/src/Results/Shortfall.jl | 6 + PRASReport.jl/src/PRASReport.jl | 5 +- PRASReport.jl/src/event_db_schema.sql | 293 +++++++++++++++++++++++++ PRASReport.jl/src/events.jl | 71 +++--- PRASReport.jl/src/report.jl | 47 ++++ PRASReport.jl/src/write.jl | 301 ++++++++++++++++++++++++++ 7 files changed, 694 insertions(+), 32 deletions(-) create mode 100644 PRASReport.jl/src/event_db_schema.sql create mode 100644 PRASReport.jl/src/report.jl create mode 100644 PRASReport.jl/src/write.jl diff --git a/PRASCore.jl/src/Results/Results.jl b/PRASCore.jl/src/Results/Results.jl index ad6f4028..7c231b45 100644 --- a/PRASCore.jl/src/Results/Results.jl +++ b/PRASCore.jl/src/Results/Results.jl @@ -70,6 +70,9 @@ EUE(x::AbstractShortfallResult, r::AbstractString, ::Colon) = EUE(x::AbstractShortfallResult, ::Colon, ::Colon) = EUE.(x, x.regions.names, permutedims(x.timestamps)) +NEUE(x::AbstractShortfallResult, ::Colon, t::ZonedDateTime) = + NEUE.(x, x.regions.names, t) + NEUE(x::AbstractShortfallResult, r::AbstractString, ::Colon) = NEUE.(x, r, x.timestamps) diff --git a/PRASCore.jl/src/Results/Shortfall.jl b/PRASCore.jl/src/Results/Shortfall.jl index e76c00c1..dc4b9462 100644 --- a/PRASCore.jl/src/Results/Shortfall.jl +++ b/PRASCore.jl/src/Results/Shortfall.jl @@ -267,6 +267,12 @@ function NEUE(x::ShortfallResult{N,L,T,E}, r::AbstractString) where {N,L,T,E} return NEUE(div(MeanEstimate(x[r]..., x.nsamples),(sum(x.regions.load[i_r,:])/1e6))) end +function NEUE(x::ShortfallResult{N,L,T,E}, r::AbstractString, t::ZonedDateTime) where {N,L,T,E} + i_r = findfirstunique(x.regions.names, r) + i_t = findfirstunique(x.timestamps, t) + return NEUE(div(MeanEstimate(x[r, t]..., x.nsamples),x.regions.load[i_r,i_t]/1e6)) +end + function finalize( acc::ShortfallAccumulator, system::SystemModel{N,L,T,P,E}, diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl index b9bd821e..b2c7c184 100644 --- a/PRASReport.jl/src/PRASReport.jl +++ b/PRASReport.jl/src/PRASReport.jl @@ -19,7 +19,10 @@ using Base64 using JSON3 include("events.jl") +include("write.jl") +include("report.jl") -export Event, get_events, event_length +export Event, get_events, event_length, sf_ts, flow_ts +export write_db!, get_db, write_regions!, write_interfaces! end # module PRASReport diff --git a/PRASReport.jl/src/event_db_schema.sql b/PRASReport.jl/src/event_db_schema.sql new file mode 100644 index 00000000..6193843e --- /dev/null +++ b/PRASReport.jl/src/event_db_schema.sql @@ -0,0 +1,293 @@ +-- PRAS Event Database Schema for DuckDB +-- This schema supports Event, sf_ts, and flow_ts data structures +-- +-- OPTIMIZATION NOTES: +-- 1. DuckDB automatically applies columnar compression (lightweight compression, dictionary encoding) +-- 2. Tables are ordered by primary key for optimal range queries +-- 3. Time-series tables are partitioned by year for better performance on temporal queries +-- 4. Compound indexes are designed for common access patterns +-- +-- RECOMMENDED SETTINGS for large datasets: +-- SET memory_limit = '8GB'; -- Adjust based on available RAM +-- SET max_memory = '16GB'; -- For complex analytical queries +-- SET threads = 4; -- Adjust based on CPU cores + +-- Parameters lookup table for default/reference configurations +CREATE TABLE parameters ( + id INTEGER PRIMARY KEY, + step_size INTEGER NOT NULL, + time_unit TEXT NOT NULL, + energy_unit TEXT DEFAULT 'MWh' NOT NULL, + + -- Constraint to ensure valid ISO 8601 duration units + CONSTRAINT valid_time_unit CHECK ( + time_unit IN ('PT1H', 'PT30M', 'PT15M', 'PT5M', 'PT1M', 'P1D', 'PT6H', 'PT12H') + ), + + -- Make parameter combinations unique + UNIQUE(step_size, time_unit, energy_unit) +); + +-- Regions lookup table +CREATE TABLE regions ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE NOT NULL +); + +-- Interfaces lookup table (region to region connections) +CREATE TABLE interfaces ( + id INTEGER PRIMARY KEY, + region_from_id INTEGER REFERENCES regions(id), + region_to_id INTEGER REFERENCES regions(id), + name TEXT, -- Optional descriptive name like "Region1->Region2" + UNIQUE(region_from_id, region_to_id) +); + +-- Main events table (clean, no parameters) +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + start_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + end_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + time_period_count INTEGER NOT NULL -- N parameter +); + +-- System-level metrics for each event (aggregated) +CREATE TABLE event_system_shortfall ( + event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + lole REAL NOT NULL, + eue REAL NOT NULL, + neue REAL NOT NULL, + PRIMARY KEY (event_id) +); + +-- Regional metrics for each event (aggregated) +CREATE TABLE event_regional_shortfall ( + id INTEGER PRIMARY KEY, + event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + region_id INTEGER REFERENCES regions(id), + lole REAL NOT NULL, + eue REAL NOT NULL, + neue REAL NOT NULL, + UNIQUE(event_id, region_id) +); + +-- Time-series metrics for each timestamp within an event (from sf_ts struct) +-- Optimized with better data types and ordering for columnar storage +CREATE TABLE event_timeseries_shortfall ( + event_id INTEGER NOT NULL, + region_id INTEGER NOT NULL, + timestamp_value TIMESTAMP WITH TIME ZONE NOT NULL, + lole REAL NOT NULL, + eue REAL NOT NULL, + neue REAL NOT NULL, + FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE, + FOREIGN KEY (region_id) REFERENCES regions(id), + PRIMARY KEY (event_id, region_id, timestamp_value) +) PARTITION BY RANGE (DATE_PART('year', timestamp_value)); + +-- Flow data for each timestamp within an event (from flow_ts struct) +-- Optimized with better data types and ordering for columnar storage +CREATE TABLE event_timeseries_flows ( + event_id INTEGER NOT NULL, + interface_id INTEGER NOT NULL, + timestamp_value TIMESTAMP WITH TIME ZONE NOT NULL, + flow REAL NOT NULL, -- Flow value (NEUE units) + FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE, + FOREIGN KEY (interface_id) REFERENCES interfaces(id), + PRIMARY KEY (event_id, interface_id, timestamp_value) +) PARTITION BY RANGE (DATE_PART('year', timestamp_value)); + +-- Optimized indexes for common access patterns +-- Compound indexes for filtering and joining patterns +CREATE INDEX idx_events_timestamps ON events(start_timestamp, end_timestamp); +CREATE INDEX idx_events_name ON events(name); -- For event name lookups + +-- Regional metrics - optimize for both directions +CREATE INDEX idx_regional_metrics_event_region ON event_regional_metrics(event_id, region_id); +CREATE INDEX idx_regional_metrics_region_event ON event_regional_metrics(region_id, event_id); + +-- Time-series metrics - optimize for common query patterns +CREATE INDEX idx_timeseries_metrics_event_time ON event_timeseries_metrics(event_id, timestamp_value); +CREATE INDEX idx_timeseries_metrics_region_time ON event_timeseries_metrics(region_id, timestamp_value); +CREATE INDEX idx_timeseries_metrics_time_region ON event_timeseries_metrics(timestamp_value, region_id); + +-- Flow metrics - optimize for interface and time queries +CREATE INDEX idx_timeseries_flows_event_time ON event_timeseries_flows(event_id, timestamp_value); +CREATE INDEX idx_timeseries_flows_interface_time ON event_timeseries_flows(interface_id, timestamp_value); + +-- Interface lookups +CREATE INDEX idx_interfaces_regions ON interfaces(region_from_id, region_to_id); +CREATE INDEX idx_interfaces_reverse ON interfaces(region_to_id, region_from_id); + +-- ============================================================================ +-- VIEWS +-- ============================================================================ + +-- Complete event summary with system and regional metrics +CREATE VIEW event_summary AS +SELECT + e.id, + e.name, + e.start_timestamp, + e.end_timestamp, + e.time_period_count, + DATE_PART('hour', e.end_timestamp - e.start_timestamp) + 1 AS event_duration_hours, + esm.lole AS system_lole, + esm.eue AS system_eue, + esm.neue AS system_neue, + COUNT(erm.region_id) AS num_regions +FROM events e +JOIN event_system_metrics esm ON e.id = esm.event_id +LEFT JOIN event_regional_metrics erm ON e.id = erm.event_id +GROUP BY e.id, e.name, e.start_timestamp, e.end_timestamp, e.time_period_count, + esm.lole, esm.eue, esm.neue; + +-- Regional breakdown for all events +CREATE VIEW event_regional_breakdown AS +SELECT + e.id AS event_id, + e.name AS event_name, + r.name AS region_name, + erm.lole, + erm.eue, + erm.neue, + ROUND(erm.eue / esm.eue * 100, 2) AS eue_percentage_of_system +FROM events e +JOIN event_regional_metrics erm ON e.id = erm.event_id +JOIN regions r ON erm.region_id = r.id +JOIN event_system_metrics esm ON e.id = esm.event_id; + +-- Time series data with event and region names +CREATE VIEW event_timeseries_detailed AS +SELECT + e.id AS event_id, + e.name AS event_name, + r.name AS region_name, + etsm.timestamp_value, + etsm.lole, + etsm.eue, + etsm.neue +FROM events e +JOIN event_timeseries_metrics etsm ON e.id = etsm.event_id +JOIN regions r ON etsm.region_id = r.id +ORDER BY e.id, r.name, etsm.timestamp_value; + +-- Flow data with interface details +CREATE VIEW event_flows_detailed AS +SELECT + e.id AS event_id, + e.name AS event_name, + rf.name AS region_from, + rt.name AS region_to, + i.name AS interface_name, + etf.timestamp_value, + etf.flow +FROM events e +JOIN event_timeseries_flows etf ON e.id = etf.event_id +JOIN interfaces i ON etf.interface_id = i.id +JOIN regions rf ON i.region_from_id = rf.id +JOIN regions rt ON i.region_to_id = rt.id +ORDER BY e.id, i.name, etf.timestamp_value; + +-- Event rankings by severity (highest EUE first) +CREATE VIEW events_by_severity AS +SELECT + e.id, + e.name, + e.start_timestamp, + e.end_timestamp, + esm.eue AS system_eue, + esm.lole AS system_lole, + esm.neue AS system_neue, + RANK() OVER (ORDER BY esm.eue DESC) AS severity_rank +FROM events e +JOIN event_system_metrics esm ON e.id = esm.event_id; + +-- Monthly event statistics +CREATE VIEW monthly_event_stats AS +SELECT + DATE_PART('year', e.start_timestamp) AS year, + DATE_PART('month', e.start_timestamp) AS month, + COUNT(*) AS event_count, + AVG(esm.eue) AS avg_eue, + SUM(esm.eue) AS total_eue, + MAX(esm.eue) AS max_eue, + AVG(DATE_PART('hour', e.end_timestamp - e.start_timestamp) + 1) AS avg_duration_hours +FROM events e +JOIN event_system_metrics esm ON e.id = esm.event_id +GROUP BY DATE_PART('year', e.start_timestamp), DATE_PART('month', e.start_timestamp) +ORDER BY year, month; + +-- Regional contribution analysis +CREATE VIEW regional_contribution_summary AS +SELECT + r.name AS region_name, + COUNT(erm.event_id) AS events_participated, + AVG(erm.eue) AS avg_regional_eue, + SUM(erm.eue) AS total_regional_eue, + AVG(erm.eue / esm.eue * 100) AS avg_contribution_percentage +FROM regions r +JOIN event_regional_metrics erm ON r.id = erm.region_id +JOIN event_system_metrics esm ON erm.event_id = esm.event_id +GROUP BY r.name +ORDER BY total_regional_eue DESC; + +-- Interface flow summary +CREATE VIEW interface_flow_summary AS +SELECT + rf.name AS region_from, + rt.name AS region_to, + i.name AS interface_name, + COUNT(etf.event_id) AS events_with_flow, + AVG(etf.flow) AS avg_flow, + MAX(etf.flow) AS max_flow, + MIN(etf.flow) AS min_flow, + SUM(etf.flow) AS total_flow +FROM interfaces i +JOIN regions rf ON i.region_from_id = rf.id +JOIN regions rt ON i.region_to_id = rt.id +JOIN event_timeseries_flows etf ON i.id = etf.interface_id +GROUP BY rf.name, rt.name, i.name +ORDER BY total_flow DESC; + +-- Event EUE matrix view - easy subsetting for specific events +CREATE VIEW event_eue_matrix AS +SELECT + e.id AS event_id, + e.name AS event_name, + etsm.timestamp_value, + r.name AS region_name, + etsm.eue +FROM events e +JOIN event_timeseries_metrics etsm ON e.id = etsm.event_id +JOIN regions r ON etsm.region_id = r.id +ORDER BY e.id, etsm.timestamp_value, r.name; + +-- ============================================================================ +-- MATERIALIZED VIEWS FOR HEAVY ANALYTICS (optional - use for frequently accessed aggregations) +-- ============================================================================ + +-- Pre-computed hourly aggregations (uncomment if needed for performance) +-- CREATE MATERIALIZED VIEW hourly_system_metrics AS +-- SELECT +-- DATE_TRUNC('hour', etsm.timestamp_value) AS hour, +-- AVG(etsm.eue) AS avg_eue, +-- SUM(etsm.eue) AS total_eue, +-- MAX(etsm.eue) AS max_eue, +-- COUNT(*) AS sample_count +-- FROM event_timeseries_metrics etsm +-- GROUP BY DATE_TRUNC('hour', etsm.timestamp_value); + +-- Pre-computed regional summaries (uncomment if needed for performance) +-- CREATE MATERIALIZED VIEW regional_monthly_summary AS +-- SELECT +-- r.name AS region_name, +-- DATE_PART('year', etsm.timestamp_value) AS year, +-- DATE_PART('month', etsm.timestamp_value) AS month, +-- AVG(etsm.eue) AS avg_eue, +-- SUM(etsm.eue) AS total_eue, +-- COUNT(DISTINCT etsm.event_id) AS events_count +-- FROM event_timeseries_metrics etsm +-- JOIN regions r ON etsm.region_id = r.id +-- GROUP BY r.name, DATE_PART('year', etsm.timestamp_value), DATE_PART('month', etsm.timestamp_value); diff --git a/PRASReport.jl/src/events.jl b/PRASReport.jl/src/events.jl index da2db82b..2a64c467 100644 --- a/PRASReport.jl/src/events.jl +++ b/PRASReport.jl/src/events.jl @@ -1,14 +1,20 @@ mutable struct Event{N,L,T,E} name::String timestamps::StepRange{ZonedDateTime,T} + system_lole::LOLE + system_eue::EUE + system_neue::NEUE lole::Vector{LOLE} eue::Vector{EUE} neue::Vector{NEUE} regions::Vector{String} function Event{}(name::String, timestamps::StepRange{ZonedDateTime,T}, + system_lole::LOLE{N,L,T}, system_eue::EUE{N,L,T,E}, + system_neue::NEUE, lole::Vector{LOLE{N,L,T}}, eue::Vector{EUE{N,L,T,E}}, - neue::Vector{NEUE}, regions::Vector{String} + neue::Vector{NEUE}, + regions::Vector{String} ) where {N,L,T,E} length(lole) != length(eue) != length(neue) != length(regions) && @@ -17,39 +23,40 @@ mutable struct Event{N,L,T,E} length(timestamps) != N && error("Number of timesteps should match metrics event length") - length(regions) > 1 && !isapprox(val(eue[1]),sum(val.(eue[2:end]))) && - error("First value in an event eue array should represent system EUE" - *"which is approximately the sum of EUE of all the regions") + length(regions) > 0 && !isapprox(val(system_eue),sum(val.(eue))) && + error("System EUE should be approximately the sum of EUE of all the regions") - new{N,L,T,E}(name, timestamps, lole, eue, neue, regions) + new{N,L,T,E}(name, timestamps, + system_lole, system_eue, system_neue, + lole, eue, neue, regions) end end -mutable struct sf_ts{} +mutable struct Shortfall_timeseries{} name::String timestamps::Vector{ZonedDateTime} - eue::Vector{Vector{EUE}} - lole::Vector{Vector{LOLE}} - neue::Vector{Vector{NEUE}} + eue::Vector{Vector{Float64}} + lole::Vector{Vector{Float64}} + neue::Vector{Vector{Float64}} regions::Vector{String} - function sf_ts(event,sf{N,L,T,E}) where {N,L,T,E} + function Shortfall_timeseries(event,sf::ShortfallResult{N,L,T,E}) where {N,L,T,E} name = event.name timestamps = collect(event.timestamps) - eue = EUE{1,L,T,E}(MeanEstimate(val.(EUE.(sf,:,timestamps)))) - lole = LOLE{1,L,T}(MeanEstimate(val.(LOLE.(sf,:,timestamps)))) - neue = NEUE(MeanEstimate(val.(NEUE.(sf,:,timestamps)))) - regions = event.regions[2:end] - new(name,timestamps,[eue],[lole],[neue],regions) + eue = map(row->val.(row),(EUE.(sf,:,timestamps))) + lole = map(row->val.(row),(LOLE.(sf,:,timestamps))) + neue = map(row->val.(row),(NEUE.(sf,:,timestamps))) + regions = event.regions + new(name,timestamps,eue,lole,neue,regions) end end -mutable struct flow_ts{} +mutable struct Flow_timeseries{} name::String timestamps::Vector{ZonedDateTime} flow::Vector{Vector{NEUE}} interfaces::Vector{String} - function flow_ts(timestamps::Vector{ZonedDateTime}) + function Flow_timeseries(timestamps::Vector{ZonedDateTime}) new(timestamps) end end @@ -70,25 +77,23 @@ function Event(sf::ShortfallResult{N,L,T,E}, ts_first = findfirstunique(sf.timestamps,first(event_timestamps)) ts_last = findfirstunique(sf.timestamps,last(event_timestamps)) - - - lole = [LOLE{event_length,L,T}( + system_lole = LOLE{event_length,L,T}( MeanEstimate(sum(val.(LOLE.(sf,event_timestamps)))) - )] + ) - eue = [EUE{event_length,L,T,E}( + system_eue = EUE{event_length,L,T,E}( MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))) - )] + ) - neue = [NEUE( + system_neue = NEUE( div(MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))), - sum(sf.regions.load[:,ts_first:ts_last])/1e6))] + sum(sf.regions.load[:,ts_first:ts_last])/1e6)) + + lole = LOLE{event_length,L,T}[] + eue = EUE{event_length,L,T,E}[] + neue = NEUE[] - if length(sf.regions) == 1 - # TODO: Change variable name - # TODO: Should all events have common region_names? - region_names = sf.regions.names - else + if length(sf.regions) > 1 region_names = ["System"] for (r,region) in enumerate(sf.regions.names) @@ -98,6 +103,7 @@ function Event(sf::ShortfallResult{N,L,T,E}, MeanEstimate(sum(val.(LOLE.(sf,region,event_timestamps)))) ) ) + push!(eue, EUE{event_length,L,T,E}( MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))) @@ -114,7 +120,10 @@ function Event(sf::ShortfallResult{N,L,T,E}, end end - return Event(name,event_timestamps,lole,eue,neue,region_names) + return Event(name,event_timestamps, + system_lole, system_eue, system_neue, + lole,eue,neue, + sf.regions.names) end """ diff --git a/PRASReport.jl/src/report.jl b/PRASReport.jl/src/report.jl new file mode 100644 index 00000000..8527406e --- /dev/null +++ b/PRASReport.jl/src/report.jl @@ -0,0 +1,47 @@ +function run_reports(sim::Simulation; kwargs...) + return assess(sim; kwargs...) +end + +""" + get_db(sf::ShortfallResult, flow::FlowResult=nothing, conn::DuckDB.Connection; kwargs...) + +Extract events from PRAS results and write them to database. +Returns the event IDs for further processing. +""" +function get_db(sf::ShortfallResult{N,L,T,E}, + conn::DuckDB.Connection, + flow::FlowResult{N,L,T,P}=nothing; + kwargs...) where {N,L,T,P,E} + + # Extract events from shortfall results + events = get_events(sf; kwargs...) + + # Write events to database (events, system metrics, regional metrics) + event_ids = write_db!(events, conn) + + # Write time-series shortfall data for each event + sf_timeseries_allevents = Shortfall_timeseries.(events, sf) + write_db!.(sf_timeseries_allevents, conn) + + # Write flow data if provided + if !isnothing(flow) + flow_timeseries_allevents = Flow_timeseries.(events, flow) # You'll need to implement this + write_db!.(flow_timeseries_allevents, conn) + end + + return event_ids +end + +""" + write_db!(events::Vector{Event}, conn::DuckDB.Connection) -> Vector{Int} + +Write a vector of events to database and return their event IDs. +""" +function write_db!(events::Vector{Event}, conn::DuckDB.Connection) + event_ids = Vector{Int}() + for event in events + event_id = write_db!(event, conn) + push!(event_ids, event_id) + end + return event_ids +end \ No newline at end of file diff --git a/PRASReport.jl/src/write.jl b/PRASReport.jl/src/write.jl new file mode 100644 index 00000000..594e16fa --- /dev/null +++ b/PRASReport.jl/src/write.jl @@ -0,0 +1,301 @@ +using DuckDB +using Dates + +""" + write_db!(events::Vector{Event}, conn::DuckDB.Connection) + +Write a vector of Event objects to the database using DuckDB Appender API for efficient bulk inserts. +Writes to: events, event_system_shortfall, event_regional_shortfall tables. +""" +function write_db!(events::Vector{Event}, conn::DuckDB.Connection) + # Write each event individually to avoid memory issues with large datasets + for event in events + write_db!(event, conn) + end +end + +""" + write_db!(event::Event, conn::DuckDB.Connection) + +Write a single Event object to the database. +""" +function write_db!(event::Event{N,L,T,E}, conn::DuckDB.Connection) where {N,L,T,E} + # Get region IDs in the same order as event.regions array + region_ids = get_region_ids_ordered(event.regions, conn) + + # Insert the event record + event_id = write_event!(event, conn) + + # Insert system-level metrics + write_system_shortfall!(event_id, event, conn) + + # Insert regional metrics if they exist + if !isempty(event.lole) && !isempty(event.eue) && !isempty(event.neue) + write_regional_shortfall!(event_id, event, region_ids, conn) + end + + return event_id +end + +""" + write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) + +Write sf_ts time-series data to event_timeseries_shortfall table. +Gets the event_id from the database using the event name for consistency. +""" +function write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) + # Get event_id from database using event name + event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", [sf_ts.name]) + if DuckDB.num_rows(event_result) == 0 + error("Event '$(sf_ts.name)' not found in database. Write the event first.") + end + event_id = DuckDB.fetchall(event_result)[1][1] + # Get region IDs in the same order as sf_ts.regions array + region_ids = get_region_ids_ordered(sf_ts.regions, conn) + + # Use Appender for efficient bulk insert + appender = DuckDB.Appender(conn, "event_timeseries_shortfall") + + try + # Iterate through timestamps and regions + for (t_idx, timestamp) in enumerate(sf_ts.timestamps) + for (r_idx, region_id) in enumerate(region_ids) + # Append row: event_id, region_id, timestamp_value, lole, eue, neue + DuckDB.append(appender, event_id) + DuckDB.append(appender, region_id) + DuckDB.append(appender, timestamp) + DuckDB.append(appender, sf_ts.lole[r_idx][t_idx]) + DuckDB.append(appender, sf_ts.eue[r_idx][t_idx]) + DuckDB.append(appender, sf_ts.neue[r_idx][t_idx]) + DuckDB.end_row(appender) + end + end + + # Flush the appender + DuckDB.flush(appender) + + finally + # Always close the appender + DuckDB.close(appender) + end +end + +""" + write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) + +Write flow_ts time-series data to event_timeseries_flows table. +Gets the event_id from the database using the event name for consistency. +""" +function write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) + # Get event_id from database using event name + event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", [flow_ts.name]) + if DuckDB.num_rows(event_result) == 0 + error("Event '$(flow_ts.name)' not found in database. Write the event first.") + end + event_id = DuckDB.fetchall(event_result)[1][1] + # Get interface IDs in the same order as flow_ts.interfaces array + interface_ids = get_interface_ids_ordered(flow_ts.interfaces, conn) + + # Use Appender for efficient bulk insert + appender = DuckDB.Appender(conn, "event_timeseries_flows") + + try + # Iterate through timestamps and interfaces + for (t_idx, timestamp) in enumerate(flow_ts.timestamps) + for (i_idx, interface_id) in enumerate(interface_ids) + # Append row: event_id, interface_id, timestamp_value, flow + DuckDB.append(appender, event_id) + DuckDB.append(appender, interface_id) + DuckDB.append(appender, timestamp) + DuckDB.append(appender, val(flow_ts.flow[i_idx][t_idx])) # Extract value from NEUE + DuckDB.end_row(appender) + end + end + + # Flush the appender + DuckDB.flush(appender) + + finally + # Always close the appender + DuckDB.close(appender) + end +end + +# ============================================================================ +# Setup Functions +# ============================================================================ + +""" + write_regions!(region_names::Vector{String}, conn::DuckDB.Connection) + +Write regions to the regions table. Call this once to populate the regions table. +Ignores regions that already exist. +""" +function write_regions!(region_names::Vector{String}, conn::DuckDB.Connection) + for region_name in region_names + try + DuckDB.execute(conn, "INSERT INTO regions (name) VALUES (?)", [region_name]) + catch e + # Region already exists, continue + if !occursin("UNIQUE constraint", string(e)) + rethrow(e) + end + end + end +end + +""" + write_interfaces!(interface_specs::Vector{Pair{String,String}}, conn::DuckDB.Connection) + +Write interfaces from region pairs to the interfaces table. +Each tuple should be (region_from, region_to). +Assumes all regions already exist in the regions table. +Call this once to populate the interfaces table. +""" +function write_interfaces!(interface_specs::Vector{Pair{String,String}}, conn::DuckDB.Connection) + for interface_pair in interface_specs + region_from, region_to = interface_pair.first, interface_pair.second + # Get region IDs + from_result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_from]) + to_result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_to]) + + if DuckDB.num_rows(from_result) == 0 + error("Region '$region_from' not found in database. Add regions first with write_regions!().") + end + if DuckDB.num_rows(to_result) == 0 + error("Region '$region_to' not found in database. Add regions first with write_regions!().") + end + + from_id = DuckDB.fetchall(from_result)[1][1] + to_id = DuckDB.fetchall(to_result)[1][1] + + interface_name = "$region_from->$region_to" + + # Insert interface (ignore if already exists due to UNIQUE constraint) + try + DuckDB.execute(conn, """ + INSERT INTO interfaces (region_from_id, region_to_id, name) + VALUES (?, ?, ?) + """, [from_id, to_id, interface_name]) + catch e + # Interface already exists, continue + if !occursin("UNIQUE constraint", string(e)) + rethrow(e) + end + end + end +end + +# ============================================================================ +# Helper Functions +# ============================================================================ + +""" + get_region_ids_ordered(region_names::Vector{String}, conn::DuckDB.Connection) -> Vector{Int} + +Get region IDs in the same order as the region_names array. +Assumes all regions exist in the database. +""" +function get_region_ids_ordered(region_names::Vector{String}, conn::DuckDB.Connection) + region_ids = Vector{Int}() + + for region_name in region_names + result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_name]) + if DuckDB.num_rows(result) == 0 + error("Region '$region_name' not found in database") + end + region_id = DuckDB.fetchall(result)[1][1] + push!(region_ids, region_id) + end + + return region_ids +end + +""" + get_interface_ids_ordered(interface_names::Vector{String}, conn::DuckDB.Connection) -> Vector{Int} + +Get interface IDs in the same order as the interface_names array. +Assumes all interfaces exist in the database. +""" +function get_interface_ids_ordered(interface_names::Vector{String}, conn::DuckDB.Connection) + interface_ids = Vector{Int}() + + for interface_name in interface_names + result = DuckDB.execute(conn, "SELECT id FROM interfaces WHERE name = ?", [interface_name]) + if DuckDB.num_rows(result) == 0 + error("Interface '$interface_name' not found in database") + end + interface_id = DuckDB.fetchall(result)[1][1] + push!(interface_ids, interface_id) + end + + return interface_ids +end + +""" + write_event!(event::Event, conn::DuckDB.Connection) -> Int + +Write event record to events table and return the event ID. +""" +function write_event!(event::Event, conn::DuckDB.Connection) + # Extract start and end timestamps + start_ts = first(event.timestamps) + end_ts = last(event.timestamps) + time_period_count = length(event.timestamps) + + # Insert event and get the ID + DuckDB.execute(conn, """ + INSERT INTO events (name, start_timestamp, end_timestamp, time_period_count) + VALUES (?, ?, ?, ?) + """, [event.name, start_ts, end_ts, time_period_count]) + + # Get the inserted event ID + result = DuckDB.execute(conn, "SELECT last_insert_rowid()") + event_id = DuckDB.fetchall(result)[1][1] + + return event_id +end + +""" + write_system_shortfall!(event_id::Int, event::Event, conn::DuckDB.Connection) + +Write system-level shortfall metrics to event_system_shortfall table. +""" +function write_system_shortfall!(event_id::Int, event::Event, conn::DuckDB.Connection) + DuckDB.execute(conn, """ + INSERT INTO event_system_shortfall (event_id, lole, eue, neue) + VALUES (?, ?, ?, ?) + """, [event_id, val(event.system_lole), val(event.system_eue), val(event.system_neue)]) +end + +""" + write_regional_shortfall!(event_id::Int, event::Event, region_ids::Vector{Int}, conn::DuckDB.Connection) + +Write regional shortfall metrics to event_regional_shortfall table. +""" +function write_regional_shortfall!(event_id::Int, event::Event, region_ids::Vector{Int}, conn::DuckDB.Connection) + # Use Appender for efficient bulk insert + appender = DuckDB.Appender(conn, "event_regional_shortfall") + + try + for (i, region_id) in enumerate(region_ids) + # Append row: event_id, region_id, lole, eue, neue + # Note: skipping the 'id' column since it's auto-generated + DuckDB.append(appender, event_id) + DuckDB.append(appender, region_id) + DuckDB.append(appender, val(event.lole[i])) + DuckDB.append(appender, val(event.eue[i])) + DuckDB.append(appender, val(event.neue[i])) + DuckDB.end_row(appender) + end + + # Flush the appender + DuckDB.flush(appender) + + finally + # Always close the appender + DuckDB.close(appender) + end +end + + From 8388f0d9144aaa4917a643c2573e43a508f745ed Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Mon, 29 Sep 2025 01:09:36 -0600 Subject: [PATCH 05/14] Working events and sf+flow timeseries DB export --- PRASReport.jl/Project.toml | 1 + PRASReport.jl/examples/run_report.jl | 6 +- PRASReport.jl/src/PRASReport.jl | 24 ++- PRASReport.jl/src/event_db_schema.sql | 106 +++------- PRASReport.jl/src/events.jl | 64 +++--- PRASReport.jl/src/report.jl | 78 ++++--- PRASReport.jl/src/{write.jl => writedb.jl} | 230 +++++++++++---------- 7 files changed, 258 insertions(+), 251 deletions(-) rename PRASReport.jl/src/{write.jl => writedb.jl} (66%) diff --git a/PRASReport.jl/Project.toml b/PRASReport.jl/Project.toml index f4372ff7..0e2ffc76 100644 --- a/PRASReport.jl/Project.toml +++ b/PRASReport.jl/Project.toml @@ -13,6 +13,7 @@ PRASCapacityCredits = "2e1a2ed5-e89d-4cd3-bc86-c0e88a73d3a3" PRASCore = "c5c32b99-e7c3-4530-a685-6f76e19f7fe2" PRASFiles = "a2806276-6d43-4ef5-91c0-491704cd7cf1" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" [compat] diff --git a/PRASReport.jl/examples/run_report.jl b/PRASReport.jl/examples/run_report.jl index 2d094777..d62e8e57 100644 --- a/PRASReport.jl/examples/run_report.jl +++ b/PRASReport.jl/examples/run_report.jl @@ -1,10 +1,11 @@ +using Revise using PRAS using PRASReport rts_sys = rts_gmlc(); rts_sys.regions.load .+= 375; -sf, = assess(rts_sys,SequentialMonteCarlo(samples=100),Shortfall()); +sf,flow = assess(rts_sys,SequentialMonteCarlo(samples=100),Shortfall(),Flow()); event_threshold = 0 events = get_events(sf,event_threshold) @@ -14,3 +15,6 @@ println("System $(NEUE(sf))") println("Number of events where each event-hour in the event has EUE > $event_threshold MW: ", length(events)) println("Longest event is over a period of ", maximum(event_length.(events))) +long_event = events[argmax(event_length.(events))] +sfts = Shortfall_timeseries.(events,sf) +flowts = Flow_timeseries(long_event,flow) \ No newline at end of file diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl index b2c7c184..1c88f2cd 100644 --- a/PRASReport.jl/src/PRASReport.jl +++ b/PRASReport.jl/src/PRASReport.jl @@ -1,28 +1,30 @@ module PRASReport import PRASCore.Systems: SystemModel, Regions, Interfaces, - Generators, Storages, GeneratorStorages, Lines, - timeunits, powerunits, energyunits, unitsymbol - + Generators, Storages, GeneratorStorages, Lines, + timeunits, powerunits, energyunits, unitsymbol, + unitsymbol_long import PRASCore.Simulations: assess - import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, FlowResult, - ShortfallSamplesResult, AbstractShortfallResult, - Result, MeanEstimate, findfirstunique, - val, stderror + ShortfallSamplesResult, AbstractShortfallResult, + Result, MeanEstimate, findfirstunique, + val, stderror import StatsBase: mean import Dates: @dateformat_str, format, now -import TimeZones: ZonedDateTime +import TimeZones: ZonedDateTime, @tz_str -using DuckDB +# TODO: Fix use only what imports are needed +import DuckDB using Base64 using JSON3 +using Dates +using Tables include("events.jl") -include("write.jl") +include("writedb.jl") include("report.jl") -export Event, get_events, event_length, sf_ts, flow_ts +export Event, get_events, event_length, Shortfall_timeseries, Flow_timeseries export write_db!, get_db, write_regions!, write_interfaces! end # module PRASReport diff --git a/PRASReport.jl/src/event_db_schema.sql b/PRASReport.jl/src/event_db_schema.sql index 6193843e..b2ee911b 100644 --- a/PRASReport.jl/src/event_db_schema.sql +++ b/PRASReport.jl/src/event_db_schema.sql @@ -1,31 +1,15 @@ --- PRAS Event Database Schema for DuckDB --- This schema supports Event, sf_ts, and flow_ts data structures --- --- OPTIMIZATION NOTES: --- 1. DuckDB automatically applies columnar compression (lightweight compression, dictionary encoding) --- 2. Tables are ordered by primary key for optimal range queries --- 3. Time-series tables are partitioned by year for better performance on temporal queries --- 4. Compound indexes are designed for common access patterns --- --- RECOMMENDED SETTINGS for large datasets: --- SET memory_limit = '8GB'; -- Adjust based on available RAM --- SET max_memory = '16GB'; -- For complex analytical queries --- SET threads = 4; -- Adjust based on CPU cores - --- Parameters lookup table for default/reference configurations +-- System and Simulation parameters CREATE TABLE parameters ( - id INTEGER PRIMARY KEY, step_size INTEGER NOT NULL, time_unit TEXT NOT NULL, - energy_unit TEXT DEFAULT 'MWh' NOT NULL, + power_unit TEXT NOT NULL, + energy_unit TEXT NOT NULL, + n_samples INTEGER, -- Constraint to ensure valid ISO 8601 duration units CONSTRAINT valid_time_unit CHECK ( - time_unit IN ('PT1H', 'PT30M', 'PT15M', 'PT5M', 'PT1M', 'P1D', 'PT6H', 'PT12H') + time_unit IN ('Year', 'Day', 'Hour', 'Minute', 'Second') ), - - -- Make parameter combinations unique - UNIQUE(step_size, time_unit, energy_unit) ); -- Regions lookup table @@ -39,13 +23,14 @@ CREATE TABLE interfaces ( id INTEGER PRIMARY KEY, region_from_id INTEGER REFERENCES regions(id), region_to_id INTEGER REFERENCES regions(id), - name TEXT, -- Optional descriptive name like "Region1->Region2" + name TEXT, -- name like "Region1->Region2" UNIQUE(region_from_id, region_to_id) ); -- Main events table (clean, no parameters) +CREATE SEQUENCE eventid_sequence START 1; CREATE TABLE events ( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY DEFAULT nextval('eventid_sequence'), name TEXT NOT NULL, start_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, end_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, @@ -54,7 +39,7 @@ CREATE TABLE events ( -- System-level metrics for each event (aggregated) CREATE TABLE event_system_shortfall ( - event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + event_id INTEGER REFERENCES events(id), lole REAL NOT NULL, eue REAL NOT NULL, neue REAL NOT NULL, @@ -64,7 +49,7 @@ CREATE TABLE event_system_shortfall ( -- Regional metrics for each event (aggregated) CREATE TABLE event_regional_shortfall ( id INTEGER PRIMARY KEY, - event_id INTEGER REFERENCES events(id) ON DELETE CASCADE, + event_id INTEGER REFERENCES events(id), region_id INTEGER REFERENCES regions(id), lole REAL NOT NULL, eue REAL NOT NULL, @@ -81,10 +66,10 @@ CREATE TABLE event_timeseries_shortfall ( lole REAL NOT NULL, eue REAL NOT NULL, neue REAL NOT NULL, - FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE, + FOREIGN KEY (event_id) REFERENCES events(id), FOREIGN KEY (region_id) REFERENCES regions(id), PRIMARY KEY (event_id, region_id, timestamp_value) -) PARTITION BY RANGE (DATE_PART('year', timestamp_value)); +); -- Flow data for each timestamp within an event (from flow_ts struct) -- Optimized with better data types and ordering for columnar storage @@ -93,10 +78,10 @@ CREATE TABLE event_timeseries_flows ( interface_id INTEGER NOT NULL, timestamp_value TIMESTAMP WITH TIME ZONE NOT NULL, flow REAL NOT NULL, -- Flow value (NEUE units) - FOREIGN KEY (event_id) REFERENCES events(id) ON DELETE CASCADE, + FOREIGN KEY (event_id) REFERENCES events(id), FOREIGN KEY (interface_id) REFERENCES interfaces(id), PRIMARY KEY (event_id, interface_id, timestamp_value) -) PARTITION BY RANGE (DATE_PART('year', timestamp_value)); +); -- Optimized indexes for common access patterns -- Compound indexes for filtering and joining patterns @@ -104,13 +89,13 @@ CREATE INDEX idx_events_timestamps ON events(start_timestamp, end_timestamp); CREATE INDEX idx_events_name ON events(name); -- For event name lookups -- Regional metrics - optimize for both directions -CREATE INDEX idx_regional_metrics_event_region ON event_regional_metrics(event_id, region_id); -CREATE INDEX idx_regional_metrics_region_event ON event_regional_metrics(region_id, event_id); +CREATE INDEX idx_regional_shortfall_event_region ON event_regional_shortfall(event_id, region_id); +CREATE INDEX idx_regional_shortfall_region_event ON event_regional_shortfall(region_id, event_id); -- Time-series metrics - optimize for common query patterns -CREATE INDEX idx_timeseries_metrics_event_time ON event_timeseries_metrics(event_id, timestamp_value); -CREATE INDEX idx_timeseries_metrics_region_time ON event_timeseries_metrics(region_id, timestamp_value); -CREATE INDEX idx_timeseries_metrics_time_region ON event_timeseries_metrics(timestamp_value, region_id); +CREATE INDEX idx_timeseries_shortfall_event_time ON event_timeseries_shortfall(event_id, timestamp_value); +CREATE INDEX idx_timeseries_shortfall_region_time ON event_timeseries_shortfall(region_id, timestamp_value); +CREATE INDEX idx_timeseries_shortfall_time_region ON event_timeseries_shortfall(timestamp_value, region_id); -- Flow metrics - optimize for interface and time queries CREATE INDEX idx_timeseries_flows_event_time ON event_timeseries_flows(event_id, timestamp_value); @@ -131,15 +116,14 @@ SELECT e.name, e.start_timestamp, e.end_timestamp, - e.time_period_count, - DATE_PART('hour', e.end_timestamp - e.start_timestamp) + 1 AS event_duration_hours, + e.time_period_count AS event_duration_hours, esm.lole AS system_lole, esm.eue AS system_eue, esm.neue AS system_neue, COUNT(erm.region_id) AS num_regions FROM events e -JOIN event_system_metrics esm ON e.id = esm.event_id -LEFT JOIN event_regional_metrics erm ON e.id = erm.event_id +JOIN event_system_shortfall esm ON e.id = esm.event_id +LEFT JOIN event_regional_shortfall erm ON e.id = erm.event_id GROUP BY e.id, e.name, e.start_timestamp, e.end_timestamp, e.time_period_count, esm.lole, esm.eue, esm.neue; @@ -154,9 +138,9 @@ SELECT erm.neue, ROUND(erm.eue / esm.eue * 100, 2) AS eue_percentage_of_system FROM events e -JOIN event_regional_metrics erm ON e.id = erm.event_id +JOIN event_regional_shortfall erm ON e.id = erm.event_id JOIN regions r ON erm.region_id = r.id -JOIN event_system_metrics esm ON e.id = esm.event_id; +JOIN event_system_shortfall esm ON e.id = esm.event_id; -- Time series data with event and region names CREATE VIEW event_timeseries_detailed AS @@ -169,7 +153,7 @@ SELECT etsm.eue, etsm.neue FROM events e -JOIN event_timeseries_metrics etsm ON e.id = etsm.event_id +JOIN event_timeseries_shortfall etsm ON e.id = etsm.event_id JOIN regions r ON etsm.region_id = r.id ORDER BY e.id, r.name, etsm.timestamp_value; @@ -202,7 +186,7 @@ SELECT esm.neue AS system_neue, RANK() OVER (ORDER BY esm.eue DESC) AS severity_rank FROM events e -JOIN event_system_metrics esm ON e.id = esm.event_id; +JOIN event_system_shortfall esm ON e.id = esm.event_id; -- Monthly event statistics CREATE VIEW monthly_event_stats AS @@ -215,7 +199,7 @@ SELECT MAX(esm.eue) AS max_eue, AVG(DATE_PART('hour', e.end_timestamp - e.start_timestamp) + 1) AS avg_duration_hours FROM events e -JOIN event_system_metrics esm ON e.id = esm.event_id +JOIN event_system_shortfall esm ON e.id = esm.event_id GROUP BY DATE_PART('year', e.start_timestamp), DATE_PART('month', e.start_timestamp) ORDER BY year, month; @@ -228,8 +212,8 @@ SELECT SUM(erm.eue) AS total_regional_eue, AVG(erm.eue / esm.eue * 100) AS avg_contribution_percentage FROM regions r -JOIN event_regional_metrics erm ON r.id = erm.region_id -JOIN event_system_metrics esm ON erm.event_id = esm.event_id +JOIN event_regional_shortfall erm ON r.id = erm.region_id +JOIN event_system_shortfall esm ON erm.event_id = esm.event_id GROUP BY r.name ORDER BY total_regional_eue DESC; @@ -260,34 +244,6 @@ SELECT r.name AS region_name, etsm.eue FROM events e -JOIN event_timeseries_metrics etsm ON e.id = etsm.event_id +JOIN event_timeseries_shortfall etsm ON e.id = etsm.event_id JOIN regions r ON etsm.region_id = r.id -ORDER BY e.id, etsm.timestamp_value, r.name; - --- ============================================================================ --- MATERIALIZED VIEWS FOR HEAVY ANALYTICS (optional - use for frequently accessed aggregations) --- ============================================================================ - --- Pre-computed hourly aggregations (uncomment if needed for performance) --- CREATE MATERIALIZED VIEW hourly_system_metrics AS --- SELECT --- DATE_TRUNC('hour', etsm.timestamp_value) AS hour, --- AVG(etsm.eue) AS avg_eue, --- SUM(etsm.eue) AS total_eue, --- MAX(etsm.eue) AS max_eue, --- COUNT(*) AS sample_count --- FROM event_timeseries_metrics etsm --- GROUP BY DATE_TRUNC('hour', etsm.timestamp_value); - --- Pre-computed regional summaries (uncomment if needed for performance) --- CREATE MATERIALIZED VIEW regional_monthly_summary AS --- SELECT --- r.name AS region_name, --- DATE_PART('year', etsm.timestamp_value) AS year, --- DATE_PART('month', etsm.timestamp_value) AS month, --- AVG(etsm.eue) AS avg_eue, --- SUM(etsm.eue) AS total_eue, --- COUNT(DISTINCT etsm.event_id) AS events_count --- FROM event_timeseries_metrics etsm --- JOIN regions r ON etsm.region_id = r.id --- GROUP BY r.name, DATE_PART('year', etsm.timestamp_value), DATE_PART('month', etsm.timestamp_value); +ORDER BY e.id, etsm.timestamp_value, r.name; \ No newline at end of file diff --git a/PRASReport.jl/src/events.jl b/PRASReport.jl/src/events.jl index 2a64c467..ed029093 100644 --- a/PRASReport.jl/src/events.jl +++ b/PRASReport.jl/src/events.jl @@ -32,6 +32,8 @@ mutable struct Event{N,L,T,E} end end +event_length(event::Event{N,L,T}) where {N,L,T} = T(N*L) + mutable struct Shortfall_timeseries{} name::String timestamps::Vector{ZonedDateTime} @@ -40,12 +42,12 @@ mutable struct Shortfall_timeseries{} neue::Vector{Vector{Float64}} regions::Vector{String} - function Shortfall_timeseries(event,sf::ShortfallResult{N,L,T,E}) where {N,L,T,E} + function Shortfall_timeseries(event,sfresult::ShortfallResult{N,L,T,E}) where {N,L,T,E} name = event.name timestamps = collect(event.timestamps) - eue = map(row->val.(row),(EUE.(sf,:,timestamps))) - lole = map(row->val.(row),(LOLE.(sf,:,timestamps))) - neue = map(row->val.(row),(NEUE.(sf,:,timestamps))) + eue = map(row->val.(row),(EUE.(sfresult,:,timestamps))) + lole = map(row->val.(row),(LOLE.(sfresult,:,timestamps))) + neue = map(row->val.(row),(NEUE.(sfresult,:,timestamps))) regions = event.regions new(name,timestamps,eue,lole,neue,regions) end @@ -54,17 +56,19 @@ end mutable struct Flow_timeseries{} name::String timestamps::Vector{ZonedDateTime} - flow::Vector{Vector{NEUE}} - interfaces::Vector{String} - function Flow_timeseries(timestamps::Vector{ZonedDateTime}) - new(timestamps) + flow::Vector{Vector{Float64}} + interfaces::Vector{Pair{String,String}} + + function Flow_timeseries(event,flresult::FlowResult{N,L,T,P}) where {N,L,T,P} + name = event.name + timestamps = collect(event.timestamps) + flow = [first.(flresult[:, ts]) for ts in timestamps] + interfaces = flresult.interfaces + new(name,timestamps,flow,interfaces) end end - -event_length(event::Event{N,L,T}) where {N,L,T} = T(N*L) - -function Event(sf::ShortfallResult{N,L,T,E}, +function Event(sfresult::ShortfallResult{N,L,T,E}, event_timestamps::StepRange{ZonedDateTime,T}, name::String=nothing ) where {N,L,T,E} @@ -74,46 +78,46 @@ function Event(sf::ShortfallResult{N,L,T,E}, end event_length = length(event_timestamps) - ts_first = findfirstunique(sf.timestamps,first(event_timestamps)) - ts_last = findfirstunique(sf.timestamps,last(event_timestamps)) + ts_first = findfirstunique(sfresult.timestamps,first(event_timestamps)) + ts_last = findfirstunique(sfresult.timestamps,last(event_timestamps)) system_lole = LOLE{event_length,L,T}( - MeanEstimate(sum(val.(LOLE.(sf,event_timestamps)))) + MeanEstimate(sum(val.(LOLE.(sfresult,event_timestamps)))) ) system_eue = EUE{event_length,L,T,E}( - MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))) + MeanEstimate(sum(val.(EUE.(sfresult,event_timestamps)))) ) system_neue = NEUE( - div(MeanEstimate(sum(val.(EUE.(sf,event_timestamps)))), - sum(sf.regions.load[:,ts_first:ts_last])/1e6)) + div(MeanEstimate(sum(val.(EUE.(sfresult,event_timestamps)))), + sum(sfresult.regions.load[:,ts_first:ts_last])/1e6)) lole = LOLE{event_length,L,T}[] eue = EUE{event_length,L,T,E}[] neue = NEUE[] - if length(sf.regions) > 1 + if length(sfresult.regions) > 1 region_names = ["System"] - for (r,region) in enumerate(sf.regions.names) + for (r,region) in enumerate(sfresult.regions.names) push!(lole, LOLE{event_length,L,T}( - MeanEstimate(sum(val.(LOLE.(sf,region,event_timestamps)))) + MeanEstimate(sum(val.(LOLE.(sfresult,region,event_timestamps)))) ) ) push!(eue, EUE{event_length,L,T,E}( - MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))) + MeanEstimate(sum(val.(EUE.(sfresult,region,event_timestamps)))) ) ) push!(neue, NEUE( - div(MeanEstimate(sum(val.(EUE.(sf,region,event_timestamps)))), - sum(sf.regions.load[r,ts_first:ts_last])/1e6)) + div(MeanEstimate(sum(val.(EUE.(sfresult,region,event_timestamps)))), + sum(sfresult.regions.load[r,ts_first:ts_last])/1e6)) ) push!(region_names,region) @@ -123,11 +127,11 @@ function Event(sf::ShortfallResult{N,L,T,E}, return Event(name,event_timestamps, system_lole, system_eue, system_neue, lole,eue,neue, - sf.regions.names) + sfresult.regions.names) end """ - get_events(sf::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} + get_events(sfresult::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} Extracts events from PRAS ShortfallResult objects where an event is a contiguous period during which the system EUE exceeds a specified threshold, and @@ -137,18 +141,18 @@ If the PRAS simulation is hourly and event_threshold is 0, and there are 5 consecutive hours where the system EUE exceeds the threshold, this returns a vector with a single event. """ -function get_events(sf::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} +function get_events(sfresult::ShortfallResult{N,L,T,E}, event_threshold=0) where {N,L,T,E} event_threshold < 0 && error("Event threshold must be non-negative") - eue_system = EUE.(sf,sf.timestamps) + eue_system = EUE.(sfresult,sfresult.timestamps) system_eue_above_threshold = findall(val.(eue_system) .> event_threshold) isempty(system_eue_above_threshold) && error("No shortfall events in this simulation") - event_timegroups = get_stepranges(sf.timestamps[system_eue_above_threshold],L,T) + event_timegroups = get_stepranges(sfresult.timestamps[system_eue_above_threshold],L,T) - return map(ts_group -> Event(sf,ts_group, + return map(ts_group -> Event(sfresult,ts_group, format(first(ts_group),"yyyy-mm-dd HH:MM ZZZ") ), event_timegroups) diff --git a/PRASReport.jl/src/report.jl b/PRASReport.jl/src/report.jl index 8527406e..bea741e1 100644 --- a/PRASReport.jl/src/report.jl +++ b/PRASReport.jl/src/report.jl @@ -1,47 +1,73 @@ -function run_reports(sim::Simulation; kwargs...) - return assess(sim; kwargs...) -end +# function run_reports(sim::Simulation; kwargs...) +# return assess(sim; kwargs...) +# end """ - get_db(sf::ShortfallResult, flow::FlowResult=nothing, conn::DuckDB.Connection; kwargs...) + get_db(sf::ShortfallResult{N,L,T,E}, + flow::FlowResult{N,L,T,P}=nothing; + conn::DuckDB.Connection=nothing, + threshold=0 Extract events from PRAS results and write them to database. Returns the event IDs for further processing. """ function get_db(sf::ShortfallResult{N,L,T,E}, - conn::DuckDB.Connection, - flow::FlowResult{N,L,T,P}=nothing; - kwargs...) where {N,L,T,P,E} + flow::Union{FlowResult{N,L,T,P},Nothing}=nothing; + conn::Union{DuckDB.Connection,Nothing}=nothing, + threshold=0) where {N,L,T,P,E} + if isnothing(conn) + timenow = Dates.format(now(tz"UTC"), @dateformat_str"yyyy-mm-dd_HHMMSSZZZ") + dbfile = DuckDB.open(joinpath(@__DIR__, "$(timenow).duckdb")) + conn = DuckDB.DBInterface.connect(dbfile) + end + + # Load in DB schema + schema_file = joinpath(dirname(@__FILE__), "event_db_schema.sql") + schema_sql = read(schema_file, String) + + # Split schema into individual statements and execute each one + # Remove SQL comments (lines starting with --) + schema_sql = join(filter(line -> !startswith(strip(line), "--"), split(schema_sql, '\n')), '\n') + statements = split(schema_sql, ';') + for stmt in statements + stmt_clean = strip(stmt) + if !isempty(stmt_clean) && !startswith(stmt_clean, "--") + try + DuckDB.DBInterface.execute(conn, stmt_clean) + catch e + println("Error executing statement: $stmt_clean") + rethrow(e) + end + end + end + + # Write system & simulation parameters to database + write_db!(sf, flow, conn) + + # Write region names to database + write_db!(sf.regions.names, conn) + # Extract events from shortfall results - events = get_events(sf; kwargs...) + events = get_events(sf,threshold) # Write events to database (events, system metrics, regional metrics) - event_ids = write_db!(events, conn) + foreach(event -> write_db!(event,conn), events) # Write time-series shortfall data for each event sf_timeseries_allevents = Shortfall_timeseries.(events, sf) - write_db!.(sf_timeseries_allevents, conn) + foreach(sf_ts -> write_db!(sf_ts,conn), sf_timeseries_allevents) # Write flow data if provided if !isnothing(flow) - flow_timeseries_allevents = Flow_timeseries.(events, flow) # You'll need to implement this - write_db!.(flow_timeseries_allevents, conn) + write_db!(flow.interfaces, conn) + flow_timeseries_allevents = Flow_timeseries.(events, flow) + foreach(flow_ts -> write_db!(flow_ts,conn), flow_timeseries_allevents) end - - return event_ids -end -""" - write_db!(events::Vector{Event}, conn::DuckDB.Connection) -> Vector{Int} + DuckDB.DBInterface.close!(conn) + DuckDB.close_database(dbfile) + + return -Write a vector of events to database and return their event IDs. -""" -function write_db!(events::Vector{Event}, conn::DuckDB.Connection) - event_ids = Vector{Int}() - for event in events - event_id = write_db!(event, conn) - push!(event_ids, event_id) - end - return event_ids end \ No newline at end of file diff --git a/PRASReport.jl/src/write.jl b/PRASReport.jl/src/writedb.jl similarity index 66% rename from PRASReport.jl/src/write.jl rename to PRASReport.jl/src/writedb.jl index 594e16fa..fe567f7b 100644 --- a/PRASReport.jl/src/write.jl +++ b/PRASReport.jl/src/writedb.jl @@ -1,6 +1,85 @@ -using DuckDB -using Dates +# ============================================================================ +# Setup Functions +# ============================================================================ +""" + write_db!(::ShortfallResult{N,L,T,E}, ::FlowResult{N,L,T,P}, conn::DuckDB.Connection) + +Write system and simulation parameters to the parameters table. +""" +function write_db!(sf::ShortfallResult{N,L,T,E}, + ::FlowResult{N,L,T,P}, + conn::DuckDB.Connection) where {N,L,T,P,E} + + try + DuckDB.execute(conn, + "INSERT INTO parameters + (step_size, time_unit, power_unit, + energy_unit, n_samples) + VALUES (?,?,?,?,?)", + [L,unitsymbol_long(T),unitsymbol(P), + unitsymbol(E),sf.nsamples]) + catch e + rethrow(e) + end +end + +""" + write_db!(region_names::Vector{String}, conn::DuckDB.Connection) + +Write regions to the regions table. Call this once to populate the regions table. +Ignores regions that already exist. +""" +function write_db!(region_names::Vector{String}, conn::DuckDB.Connection) + for (idx,region_name) in enumerate(region_names) + try + DuckDB.execute(conn, "INSERT INTO regions (id, name) VALUES (?,?)", [idx,region_name]) + catch e + # Region already exists, continue + if !occursin("UNIQUE constraint", string(e)) + rethrow(e) + end + end + end +end + +""" + write_db!(interfaces::Vector{Pair{String,String}}, conn::DuckDB.Connection) + +Write interfaces from region pairs to the interfaces table. +Each tuple should be (region_from, region_to). +Assumes all regions already exist in the regions table. +Call this once to populate the interfaces table. +""" +function write_db!(interfaces::Vector{Pair{String,String}}, conn::DuckDB.Connection) + for (idx,interface_pair) in enumerate(interfaces) + region_from, region_to = interface_pair.first, interface_pair.second + # Get region IDs + interface_reg_ids = DuckDB.execute(conn, "SELECT id FROM regions WHERE name IN [?,?]", + [region_from,region_to] + ) |> Tables.columntable + + from_id,to_id = [interface_reg_ids.id...] + + interface_name = "$region_from->$region_to" + + # Insert interface (ignore if already exists due to UNIQUE constraint) + try + DuckDB.execute(conn, """ + INSERT INTO interfaces (id, region_from_id, region_to_id, name) + VALUES (?, ?, ?, ?) + """, [idx, from_id, to_id, interface_name]) + catch e + # Interface already exists, continue + if !occursin("UNIQUE constraint", string(e)) + rethrow(e) + end + end + end +end +# ============================================================================ +# Write Results +# ============================================================================ """ write_db!(events::Vector{Event}, conn::DuckDB.Connection) @@ -45,11 +124,12 @@ Gets the event_id from the database using the event name for consistency. """ function write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) # Get event_id from database using event name - event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", [sf_ts.name]) - if DuckDB.num_rows(event_result) == 0 + event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", + [sf_ts.name]) |> Tables.columntable + isempty(event_result) && error("Event '$(sf_ts.name)' not found in database. Write the event first.") - end - event_id = DuckDB.fetchall(event_result)[1][1] + event_id = first(event_result.id) + # Get region IDs in the same order as sf_ts.regions array region_ids = get_region_ids_ordered(sf_ts.regions, conn) @@ -63,10 +143,10 @@ function write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) # Append row: event_id, region_id, timestamp_value, lole, eue, neue DuckDB.append(appender, event_id) DuckDB.append(appender, region_id) - DuckDB.append(appender, timestamp) - DuckDB.append(appender, sf_ts.lole[r_idx][t_idx]) - DuckDB.append(appender, sf_ts.eue[r_idx][t_idx]) - DuckDB.append(appender, sf_ts.neue[r_idx][t_idx]) + DuckDB.append(appender, Dates.DateTime(timestamp)) + DuckDB.append(appender, sf_ts.lole[t_idx][r_idx]) + DuckDB.append(appender, sf_ts.eue[t_idx][r_idx]) + DuckDB.append(appender, sf_ts.neue[t_idx][r_idx]) DuckDB.end_row(appender) end end @@ -81,18 +161,19 @@ function write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) end """ - write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) + write_db!(flow_ts::flow_ts, event_id::Integer, conn::DuckDB.Connection) Write flow_ts time-series data to event_timeseries_flows table. -Gets the event_id from the database using the event name for consistency. """ function write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) + # Get event_id from database using event name - event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", [flow_ts.name]) - if DuckDB.num_rows(event_result) == 0 + event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", + [flow_ts.name]) |> Tables.columntable + isempty(event_result) && error("Event '$(flow_ts.name)' not found in database. Write the event first.") - end - event_id = DuckDB.fetchall(event_result)[1][1] + event_id = first(event_result.id) + # Get interface IDs in the same order as flow_ts.interfaces array interface_ids = get_interface_ids_ordered(flow_ts.interfaces, conn) @@ -106,8 +187,8 @@ function write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) # Append row: event_id, interface_id, timestamp_value, flow DuckDB.append(appender, event_id) DuckDB.append(appender, interface_id) - DuckDB.append(appender, timestamp) - DuckDB.append(appender, val(flow_ts.flow[i_idx][t_idx])) # Extract value from NEUE + DuckDB.append(appender, Dates.DateTime(timestamp)) + DuckDB.append(appender, flow_ts.flow[t_idx][i_idx]) # Extract value from NEUE DuckDB.end_row(appender) end end @@ -121,71 +202,6 @@ function write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) end end -# ============================================================================ -# Setup Functions -# ============================================================================ - -""" - write_regions!(region_names::Vector{String}, conn::DuckDB.Connection) - -Write regions to the regions table. Call this once to populate the regions table. -Ignores regions that already exist. -""" -function write_regions!(region_names::Vector{String}, conn::DuckDB.Connection) - for region_name in region_names - try - DuckDB.execute(conn, "INSERT INTO regions (name) VALUES (?)", [region_name]) - catch e - # Region already exists, continue - if !occursin("UNIQUE constraint", string(e)) - rethrow(e) - end - end - end -end - -""" - write_interfaces!(interface_specs::Vector{Pair{String,String}}, conn::DuckDB.Connection) - -Write interfaces from region pairs to the interfaces table. -Each tuple should be (region_from, region_to). -Assumes all regions already exist in the regions table. -Call this once to populate the interfaces table. -""" -function write_interfaces!(interface_specs::Vector{Pair{String,String}}, conn::DuckDB.Connection) - for interface_pair in interface_specs - region_from, region_to = interface_pair.first, interface_pair.second - # Get region IDs - from_result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_from]) - to_result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_to]) - - if DuckDB.num_rows(from_result) == 0 - error("Region '$region_from' not found in database. Add regions first with write_regions!().") - end - if DuckDB.num_rows(to_result) == 0 - error("Region '$region_to' not found in database. Add regions first with write_regions!().") - end - - from_id = DuckDB.fetchall(from_result)[1][1] - to_id = DuckDB.fetchall(to_result)[1][1] - - interface_name = "$region_from->$region_to" - - # Insert interface (ignore if already exists due to UNIQUE constraint) - try - DuckDB.execute(conn, """ - INSERT INTO interfaces (region_from_id, region_to_id, name) - VALUES (?, ?, ?) - """, [from_id, to_id, interface_name]) - catch e - # Interface already exists, continue - if !occursin("UNIQUE constraint", string(e)) - rethrow(e) - end - end - end -end - # ============================================================================ # Helper Functions # ============================================================================ @@ -200,14 +216,11 @@ function get_region_ids_ordered(region_names::Vector{String}, conn::DuckDB.Conne region_ids = Vector{Int}() for region_name in region_names - result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_name]) - if DuckDB.num_rows(result) == 0 - error("Region '$region_name' not found in database") - end - region_id = DuckDB.fetchall(result)[1][1] - push!(region_ids, region_id) + result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_name]) |> Tables.columntable + isempty(result) && error("Region '$region_name' not found in database") + push!(region_ids, first(result.id)) end - + return region_ids end @@ -217,16 +230,16 @@ end Get interface IDs in the same order as the interface_names array. Assumes all interfaces exist in the database. """ -function get_interface_ids_ordered(interface_names::Vector{String}, conn::DuckDB.Connection) +function get_interface_ids_ordered(interface_names::Vector{Pair{String,String}}, conn::DuckDB.Connection) interface_ids = Vector{Int}() for interface_name in interface_names - result = DuckDB.execute(conn, "SELECT id FROM interfaces WHERE name = ?", [interface_name]) - if DuckDB.num_rows(result) == 0 + iname_db = "$(interface_name.first)->$(interface_name.second)" + result = DuckDB.execute(conn, "SELECT id FROM interfaces WHERE name = ?", [iname_db]) |> Tables.columntable + isempty(result) && error("Interface '$interface_name' not found in database") - end - interface_id = DuckDB.fetchall(result)[1][1] - push!(interface_ids, interface_id) + + push!(interface_ids, first(result.id)) end return interface_ids @@ -239,19 +252,20 @@ Write event record to events table and return the event ID. """ function write_event!(event::Event, conn::DuckDB.Connection) # Extract start and end timestamps - start_ts = first(event.timestamps) - end_ts = last(event.timestamps) + start_ts = Dates.DateTime(first(event.timestamps)) + end_ts = Dates.DateTime(last(event.timestamps)) time_period_count = length(event.timestamps) - # Insert event and get the ID - DuckDB.execute(conn, """ - INSERT INTO events (name, start_timestamp, end_timestamp, time_period_count) - VALUES (?, ?, ?, ?) - """, [event.name, start_ts, end_ts, time_period_count]) + # Insert event and get the ID using RETURNING clause + result = DuckDB.execute(conn, + """ + INSERT INTO events (name, start_timestamp, end_timestamp, time_period_count) + VALUES (?, ?, ?, ?) + RETURNING id + """, + [event.name, start_ts, end_ts, time_period_count]) |> Tables.columntable - # Get the inserted event ID - result = DuckDB.execute(conn, "SELECT last_insert_rowid()") - event_id = DuckDB.fetchall(result)[1][1] + event_id = first(result.id) return event_id end @@ -261,7 +275,7 @@ end Write system-level shortfall metrics to event_system_shortfall table. """ -function write_system_shortfall!(event_id::Int, event::Event, conn::DuckDB.Connection) +function write_system_shortfall!(event_id::Int32, event::Event, conn::DuckDB.Connection) DuckDB.execute(conn, """ INSERT INTO event_system_shortfall (event_id, lole, eue, neue) VALUES (?, ?, ?, ?) @@ -273,7 +287,7 @@ end Write regional shortfall metrics to event_regional_shortfall table. """ -function write_regional_shortfall!(event_id::Int, event::Event, region_ids::Vector{Int}, conn::DuckDB.Connection) +function write_regional_shortfall!(event_id::Int32, event::Event, region_ids::Vector{Int}, conn::DuckDB.Connection) # Use Appender for efficient bulk insert appender = DuckDB.Appender(conn, "event_regional_shortfall") From d18191331b2dd89a38b237977649143d25c1ec15 Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Mon, 29 Sep 2025 16:08:24 -0600 Subject: [PATCH 06/14] Have a working html template which shows the events table for the system level result --- PRASReport.jl/Project.toml | 2 - PRASReport.jl/TODOs.md | 4 +- PRASReport.jl/examples/run_report.jl | 12 +- PRASReport.jl/src/PRASReport.jl | 11 +- PRASReport.jl/src/report.jl | 82 ++--- PRASReport.jl/src/report_template.html | 475 +++++++++++++++++++++++++ PRASReport.jl/src/writedb.jl | 96 ++++- PRASReport.jl/test/runtests.jl | 2 + 8 files changed, 592 insertions(+), 92 deletions(-) create mode 100644 PRASReport.jl/src/report_template.html diff --git a/PRASReport.jl/Project.toml b/PRASReport.jl/Project.toml index 0e2ffc76..b73fcc87 100644 --- a/PRASReport.jl/Project.toml +++ b/PRASReport.jl/Project.toml @@ -7,7 +7,6 @@ version = "0.1.0" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DuckDB = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" -JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" PRAS = "05348d26-1c52-11e9-35e3-9d51842d34b9" PRASCapacityCredits = "2e1a2ed5-e89d-4cd3-bc86-c0e88a73d3a3" PRASCore = "c5c32b99-e7c3-4530-a685-6f76e19f7fe2" @@ -18,7 +17,6 @@ TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" [compat] Dates = "1" -JSON3 = "1.14" PRASCore = "0.8.0" StatsBase = "0.34" TimeZones = "^1.14" diff --git a/PRASReport.jl/TODOs.md b/PRASReport.jl/TODOs.md index 8cf068f7..91d2a98b 100644 --- a/PRASReport.jl/TODOs.md +++ b/PRASReport.jl/TODOs.md @@ -1,5 +1,4 @@ # TODO -0. Store DB as base64, or data URL in the html 1. Landing page shows most important characteristics like - Number of events with total EUE exceeds the ## number - Same with LOLE @@ -10,4 +9,5 @@ 4. Event selector creates a tab for an event to show regional shortfall and flows etc 5. Does NEUE have to be reported as MWh/MWh 6. Explore how a directed graph can be stored in DuckDB, and how it can be drawn on a webpage with WASM-DuckDB -7. Julia indentation \ No newline at end of file +7. Julia indentation +8. Better names for Shortfall_timeseries and Flow_timeseries? \ No newline at end of file diff --git a/PRASReport.jl/examples/run_report.jl b/PRASReport.jl/examples/run_report.jl index d62e8e57..38ff206e 100644 --- a/PRASReport.jl/examples/run_report.jl +++ b/PRASReport.jl/examples/run_report.jl @@ -7,14 +7,4 @@ rts_sys.regions.load .+= 375; sf,flow = assess(rts_sys,SequentialMonteCarlo(samples=100),Shortfall(),Flow()); -event_threshold = 0 -events = get_events(sf,event_threshold) - -println("System $(EUE(sf))") -println("System $(NEUE(sf))") -println("Number of events where each event-hour in the event has EUE > $event_threshold MW: ", length(events)) -println("Longest event is over a period of ", maximum(event_length.(events))) - -long_event = events[argmax(event_length.(events))] -sfts = Shortfall_timeseries.(events,sf) -flowts = Flow_timeseries(long_event,flow) \ No newline at end of file +create_html_report(sf,flow; report_name="rts_report", threshold=0) \ No newline at end of file diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl index 1c88f2cd..98cdaf75 100644 --- a/PRASReport.jl/src/PRASReport.jl +++ b/PRASReport.jl/src/PRASReport.jl @@ -10,15 +10,11 @@ import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, FlowResult, Result, MeanEstimate, findfirstunique, val, stderror import StatsBase: mean -import Dates: @dateformat_str, format, now +import Dates: @dateformat_str, format, now, DateTime import TimeZones: ZonedDateTime, @tz_str - -# TODO: Fix use only what imports are needed +import Base64: base64encode +import Tables: columntable import DuckDB -using Base64 -using JSON3 -using Dates -using Tables include("events.jl") include("writedb.jl") @@ -26,5 +22,6 @@ include("report.jl") export Event, get_events, event_length, Shortfall_timeseries, Flow_timeseries export write_db!, get_db, write_regions!, write_interfaces! +export create_html_report end # module PRASReport diff --git a/PRASReport.jl/src/report.jl b/PRASReport.jl/src/report.jl index bea741e1..64bc2dfe 100644 --- a/PRASReport.jl/src/report.jl +++ b/PRASReport.jl/src/report.jl @@ -3,71 +3,33 @@ # end """ - get_db(sf::ShortfallResult{N,L,T,E}, - flow::FlowResult{N,L,T,P}=nothing; - conn::DuckDB.Connection=nothing, - threshold=0 - -Extract events from PRAS results and write them to database. -Returns the event IDs for further processing. + create_html_report(conn::DuckDB.DuckDBConnection; + output_path::String="report.html", + title::String="PRAS Report") """ -function get_db(sf::ShortfallResult{N,L,T,E}, - flow::Union{FlowResult{N,L,T,P},Nothing}=nothing; - conn::Union{DuckDB.Connection,Nothing}=nothing, - threshold=0) where {N,L,T,P,E} - - if isnothing(conn) - timenow = Dates.format(now(tz"UTC"), @dateformat_str"yyyy-mm-dd_HHMMSSZZZ") - dbfile = DuckDB.open(joinpath(@__DIR__, "$(timenow).duckdb")) - conn = DuckDB.DBInterface.connect(dbfile) - end - - # Load in DB schema - schema_file = joinpath(dirname(@__FILE__), "event_db_schema.sql") - schema_sql = read(schema_file, String) - - # Split schema into individual statements and execute each one - # Remove SQL comments (lines starting with --) - schema_sql = join(filter(line -> !startswith(strip(line), "--"), split(schema_sql, '\n')), '\n') - statements = split(schema_sql, ';') - for stmt in statements - stmt_clean = strip(stmt) - if !isempty(stmt_clean) && !startswith(stmt_clean, "--") - try - DuckDB.DBInterface.execute(conn, stmt_clean) - catch e - println("Error executing statement: $stmt_clean") - rethrow(e) - end - end - end - - # Write system & simulation parameters to database - write_db!(sf, flow, conn) +function create_html_report(sf::ShortfallResult, + flow::FlowResult; + report_name::String="report", + threshold::Int=0) - # Write region names to database - write_db!(sf.regions.names, conn) + tempdb_path = tempname() * ".db" + dbfile = DuckDB.open(tempdb_path) + conn = DuckDB.connect(dbfile) - # Extract events from shortfall results - events = get_events(sf,threshold) - - # Write events to database (events, system metrics, regional metrics) - foreach(event -> write_db!(event,conn), events) - - # Write time-series shortfall data for each event - sf_timeseries_allevents = Shortfall_timeseries.(events, sf) - foreach(sf_ts -> write_db!(sf_ts,conn), sf_timeseries_allevents) - - # Write flow data if provided - if !isnothing(flow) - write_db!(flow.interfaces, conn) - flow_timeseries_allevents = Flow_timeseries.(events, flow) - foreach(flow_ts -> write_db!(flow_ts,conn), flow_timeseries_allevents) - end + conn = get_db(sf, flow; conn, threshold=threshold) DuckDB.DBInterface.close!(conn) DuckDB.close_database(dbfile) - return + base64_db = base64encode(read(tempdb_path)) + rm(tempdb_path; force=true) + + report_html = read(joinpath(@__DIR__, "report_template.html"), String) + report_html = replace(report_html, + " // Placeholder for base64 database - this will be replaced by Julia" => "") + report_html = replace(report_html, + "const BASE64_DB = \"{{BASE64_DB_PLACEHOLDER}}\"" => "const BASE64_DB = \"$(base64_db)\"") + + write(report_name * ".html", report_html); -end \ No newline at end of file +end \ No newline at end of file diff --git a/PRASReport.jl/src/report_template.html b/PRASReport.jl/src/report_template.html new file mode 100644 index 00000000..a267c49a --- /dev/null +++ b/PRASReport.jl/src/report_template.html @@ -0,0 +1,475 @@ + + + + + + PRAS Report + + + +
+
+

PRAS Reliability Assessment Report

+
+ +
+

Simulation Summary

+
+ - +
Samples
+
+
+ - +
Events
+
+
+ +
+
+ Reliability Events +
+
+ +

Loading data...

+
+ + + +
+
+ + + + \ No newline at end of file diff --git a/PRASReport.jl/src/writedb.jl b/PRASReport.jl/src/writedb.jl index fe567f7b..bf5316fa 100644 --- a/PRASReport.jl/src/writedb.jl +++ b/PRASReport.jl/src/writedb.jl @@ -1,3 +1,79 @@ +""" + get_db(sf::ShortfallResult{N,L,T,E}, + flow::FlowResult{N,L,T,P}=nothing; + conn::DuckDB.Connection=nothing, + threshold=0 + +Extract events from PRAS results and write them to database. +Returns the event IDs for further processing. +""" +function get_db(sf::ShortfallResult{N,L,T,E}, + flow::Union{FlowResult{N,L,T,P},Nothing}=nothing; + conn::Union{DuckDB.Connection,Nothing}=nothing, + threshold=0) where {N,L,T,P,E} + + if isnothing(conn) + timenow = format(now(tz"UTC"), @dateformat_str"yyyy-mm-dd_HHMMSSZZZ") + dbfile = DuckDB.open(joinpath(@__DIR__, "$(timenow).duckdb")) + conn = DuckDB.connect(dbfile) + internal_conn = true + else + internal_conn = false + end + + + # Load in DB schema + schema_file = joinpath(dirname(@__FILE__), "event_db_schema.sql") + schema_sql = read(schema_file, String) + + # Split schema into individual statements and execute each one + # Remove SQL comments (lines starting with --) + schema_sql = join(filter(line -> !startswith(strip(line), "--"), split(schema_sql, '\n')), '\n') + statements = split(schema_sql, ';') + for stmt in statements + stmt_clean = strip(stmt) + if !isempty(stmt_clean) && !startswith(stmt_clean, "--") + try + DuckDB.DBInterface.execute(conn, stmt_clean) + catch e + println("Error executing statement: $stmt_clean") + rethrow(e) + end + end + end + + # Write system & simulation parameters to database + write_db!(sf, flow, conn) + + # Write region names to database + write_db!(sf.regions.names, conn) + + # Extract events from shortfall results + events = get_events(sf,threshold) + + # Write events to database (events, system metrics, regional metrics) + foreach(event -> write_db!(event,conn), events) + + # Write time-series shortfall data for each event + sf_timeseries_allevents = Shortfall_timeseries.(events, sf) + foreach(sf_ts -> write_db!(sf_ts,conn), sf_timeseries_allevents) + + # Write flow data if provided + if !isnothing(flow) + write_db!(flow.interfaces, conn) + flow_timeseries_allevents = Flow_timeseries.(events, flow) + foreach(flow_ts -> write_db!(flow_ts,conn), flow_timeseries_allevents) + end + + if internal_conn + DuckDB.DBInterface.close!(conn) + DuckDB.close_database(dbfile) + return + else + return conn + end + +end # ============================================================================ # Setup Functions # ============================================================================ @@ -56,7 +132,7 @@ function write_db!(interfaces::Vector{Pair{String,String}}, conn::DuckDB.Connect # Get region IDs interface_reg_ids = DuckDB.execute(conn, "SELECT id FROM regions WHERE name IN [?,?]", [region_from,region_to] - ) |> Tables.columntable + ) |> columntable from_id,to_id = [interface_reg_ids.id...] @@ -125,7 +201,7 @@ Gets the event_id from the database using the event name for consistency. function write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) # Get event_id from database using event name event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", - [sf_ts.name]) |> Tables.columntable + [sf_ts.name]) |> columntable isempty(event_result) && error("Event '$(sf_ts.name)' not found in database. Write the event first.") event_id = first(event_result.id) @@ -143,7 +219,7 @@ function write_db!(sf_ts::Shortfall_timeseries, conn::DuckDB.Connection) # Append row: event_id, region_id, timestamp_value, lole, eue, neue DuckDB.append(appender, event_id) DuckDB.append(appender, region_id) - DuckDB.append(appender, Dates.DateTime(timestamp)) + DuckDB.append(appender, DateTime(timestamp)) DuckDB.append(appender, sf_ts.lole[t_idx][r_idx]) DuckDB.append(appender, sf_ts.eue[t_idx][r_idx]) DuckDB.append(appender, sf_ts.neue[t_idx][r_idx]) @@ -169,7 +245,7 @@ function write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) # Get event_id from database using event name event_result = DuckDB.execute(conn, "SELECT id FROM events WHERE name = ?", - [flow_ts.name]) |> Tables.columntable + [flow_ts.name]) |> columntable isempty(event_result) && error("Event '$(flow_ts.name)' not found in database. Write the event first.") event_id = first(event_result.id) @@ -187,7 +263,7 @@ function write_db!(flow_ts::Flow_timeseries, conn::DuckDB.Connection) # Append row: event_id, interface_id, timestamp_value, flow DuckDB.append(appender, event_id) DuckDB.append(appender, interface_id) - DuckDB.append(appender, Dates.DateTime(timestamp)) + DuckDB.append(appender, DateTime(timestamp)) DuckDB.append(appender, flow_ts.flow[t_idx][i_idx]) # Extract value from NEUE DuckDB.end_row(appender) end @@ -216,7 +292,7 @@ function get_region_ids_ordered(region_names::Vector{String}, conn::DuckDB.Conne region_ids = Vector{Int}() for region_name in region_names - result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_name]) |> Tables.columntable + result = DuckDB.execute(conn, "SELECT id FROM regions WHERE name = ?", [region_name]) |> columntable isempty(result) && error("Region '$region_name' not found in database") push!(region_ids, first(result.id)) end @@ -235,7 +311,7 @@ function get_interface_ids_ordered(interface_names::Vector{Pair{String,String}}, for interface_name in interface_names iname_db = "$(interface_name.first)->$(interface_name.second)" - result = DuckDB.execute(conn, "SELECT id FROM interfaces WHERE name = ?", [iname_db]) |> Tables.columntable + result = DuckDB.execute(conn, "SELECT id FROM interfaces WHERE name = ?", [iname_db]) |> columntable isempty(result) && error("Interface '$interface_name' not found in database") @@ -252,8 +328,8 @@ Write event record to events table and return the event ID. """ function write_event!(event::Event, conn::DuckDB.Connection) # Extract start and end timestamps - start_ts = Dates.DateTime(first(event.timestamps)) - end_ts = Dates.DateTime(last(event.timestamps)) + start_ts = DateTime(first(event.timestamps)) + end_ts = DateTime(last(event.timestamps)) time_period_count = length(event.timestamps) # Insert event and get the ID using RETURNING clause @@ -263,7 +339,7 @@ function write_event!(event::Event, conn::DuckDB.Connection) VALUES (?, ?, ?, ?) RETURNING id """, - [event.name, start_ts, end_ts, time_period_count]) |> Tables.columntable + [event.name, start_ts, end_ts, time_period_count]) |> columntable event_id = first(result.id) diff --git a/PRASReport.jl/test/runtests.jl b/PRASReport.jl/test/runtests.jl index 69afac67..d394d49e 100644 --- a/PRASReport.jl/test/runtests.jl +++ b/PRASReport.jl/test/runtests.jl @@ -3,6 +3,8 @@ using PRASReport using TimeZones using Dates +# Test copperplate +# Test get functions from database for wrong names @testset "PRASReport.jl Tests" begin @testset "Test events" begin From 500b46618454b41185ffc48350ca5dad72fba60c Mon Sep 17 00:00:00 2001 From: Srihari Sundar Date: Mon, 29 Sep 2025 21:30:03 -0600 Subject: [PATCH 07/14] Change how timestamps are saved, add few more elements to webpage --- PRASReport.jl/TODOs.md | 4 +- PRASReport.jl/src/PRASReport.jl | 2 +- PRASReport.jl/src/event_db_schema.sql | 23 +- PRASReport.jl/src/report.jl | 4 +- PRASReport.jl/src/report_template.html | 294 +++++++++++++++++++------ PRASReport.jl/src/writedb.jl | 42 +++- 6 files changed, 283 insertions(+), 86 deletions(-) diff --git a/PRASReport.jl/TODOs.md b/PRASReport.jl/TODOs.md index 91d2a98b..be5b3e2d 100644 --- a/PRASReport.jl/TODOs.md +++ b/PRASReport.jl/TODOs.md @@ -1,4 +1,5 @@ # TODO +0. Verify timezone handling is correct. SO CONFUSING! 1. Landing page shows most important characteristics like - Number of events with total EUE exceeds the ## number - Same with LOLE @@ -10,4 +11,5 @@ 5. Does NEUE have to be reported as MWh/MWh 6. Explore how a directed graph can be stored in DuckDB, and how it can be drawn on a webpage with WASM-DuckDB 7. Julia indentation -8. Better names for Shortfall_timeseries and Flow_timeseries? \ No newline at end of file +8. Better names for Shortfall_timeseries and Flow_timeseries? +9. Update glossary with event period definitions diff --git a/PRASReport.jl/src/PRASReport.jl b/PRASReport.jl/src/PRASReport.jl index 98cdaf75..a26d1932 100644 --- a/PRASReport.jl/src/PRASReport.jl +++ b/PRASReport.jl/src/PRASReport.jl @@ -11,7 +11,7 @@ import PRASCore.Results: EUE, LOLE, NEUE, ShortfallResult, FlowResult, val, stderror import StatsBase: mean import Dates: @dateformat_str, format, now, DateTime -import TimeZones: ZonedDateTime, @tz_str +import TimeZones: ZonedDateTime, @tz_str, TimeZone import Base64: base64encode import Tables: columntable import DuckDB diff --git a/PRASReport.jl/src/event_db_schema.sql b/PRASReport.jl/src/event_db_schema.sql index b2ee911b..3214cccd 100644 --- a/PRASReport.jl/src/event_db_schema.sql +++ b/PRASReport.jl/src/event_db_schema.sql @@ -1,15 +1,26 @@ -- System and Simulation parameters -CREATE TABLE parameters ( +CREATE TABLE systemsiminfo ( + timesteps INTEGER, step_size INTEGER NOT NULL, time_unit TEXT NOT NULL, power_unit TEXT NOT NULL, energy_unit TEXT NOT NULL, + start_timestamp TIMESTAMP WITHOUT TIME ZONE, + end_timestamp TIMESTAMP WITHOUT TIME ZONE, + timezone TEXT, n_samples INTEGER, + eue_mean REAL NOT NULL, + eue_stderr REAL NOT NULL, + lole_mean REAL NOT NULL, + lole_stderr REAL NOT NULL, + neue_mean REAL NOT NULL, + neue_stderr REAL NOT NULL, + eventthreshold INTEGER NOT NULL, -- Constraint to ensure valid ISO 8601 duration units CONSTRAINT valid_time_unit CHECK ( time_unit IN ('Year', 'Day', 'Hour', 'Minute', 'Second') - ), + ) ); -- Regions lookup table @@ -32,8 +43,8 @@ CREATE SEQUENCE eventid_sequence START 1; CREATE TABLE events ( id INTEGER PRIMARY KEY DEFAULT nextval('eventid_sequence'), name TEXT NOT NULL, - start_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - end_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + start_timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, + end_timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, time_period_count INTEGER NOT NULL -- N parameter ); @@ -62,7 +73,7 @@ CREATE TABLE event_regional_shortfall ( CREATE TABLE event_timeseries_shortfall ( event_id INTEGER NOT NULL, region_id INTEGER NOT NULL, - timestamp_value TIMESTAMP WITH TIME ZONE NOT NULL, + timestamp_value TIMESTAMP WITHOUT TIME ZONE NOT NULL, lole REAL NOT NULL, eue REAL NOT NULL, neue REAL NOT NULL, @@ -76,7 +87,7 @@ CREATE TABLE event_timeseries_shortfall ( CREATE TABLE event_timeseries_flows ( event_id INTEGER NOT NULL, interface_id INTEGER NOT NULL, - timestamp_value TIMESTAMP WITH TIME ZONE NOT NULL, + timestamp_value TIMESTAMP WITHOUT TIME ZONE NOT NULL, flow REAL NOT NULL, -- Flow value (NEUE units) FOREIGN KEY (event_id) REFERENCES events(id), FOREIGN KEY (interface_id) REFERENCES interfaces(id), diff --git a/PRASReport.jl/src/report.jl b/PRASReport.jl/src/report.jl index 64bc2dfe..ddaee75a 100644 --- a/PRASReport.jl/src/report.jl +++ b/PRASReport.jl/src/report.jl @@ -30,6 +30,8 @@ function create_html_report(sf::ShortfallResult, report_html = replace(report_html, "const BASE64_DB = \"{{BASE64_DB_PLACEHOLDER}}\"" => "const BASE64_DB = \"$(base64_db)\"") - write(report_name * ".html", report_html); + write(report_name * ".html", report_html) + + return end \ No newline at end of file diff --git a/PRASReport.jl/src/report_template.html b/PRASReport.jl/src/report_template.html index a267c49a..e787ad2a 100644 --- a/PRASReport.jl/src/report_template.html +++ b/PRASReport.jl/src/report_template.html @@ -54,27 +54,53 @@ font-size: 1.5rem; } - .stat { - display: inline-block; - background: #f8f9fa; - padding: 1rem; - border-radius: 6px; - margin-right: 1rem; - min-width: 150px; - text-align: center; + .summary-list { + list-style: none; + padding: 0; + margin: 0; } - .stat-value { - font-size: 2rem; - font-weight: bold; + .summary-list li { + padding: 0.5rem 0; + border-bottom: 1px solid #eee; + } + + .summary-list li:last-child { + border-bottom: none; + } + + .summary-list strong { color: #667eea; - display: block; + margin-right: 0.5rem; } - .stat-label { + .card-footer { + margin-top: 1rem; + padding-top: 1rem; + border-top: 1px solid #eee; + display: flex; + justify-content: space-between; + align-items: center; + font-size: 1rem; color: #666; - font-size: 0.9rem; - margin-top: 0.5rem; + } + + .card-footer a { + color: #667eea; + text-decoration: none; + font-weight: 500; + } + + .card-footer a:hover { + text-decoration: underline; + } + + .glossary-link { + text-align: left; + } + + .generated-with { + text-align: right; } .table-container { @@ -168,6 +194,57 @@ color: #667eea; } + .navigation-link { + display: inline-block; + background: #667eea; + color: white; + padding: 0.5rem 1rem; + border-radius: 4px; + text-decoration: none; + font-weight: 500; + margin-top: 1rem; + transition: background-color 0.2s; + } + + .navigation-link:hover { + background: #5a67d8; + text-decoration: none; + color: white; + } + + .page { + display: none; + } + + .page.active { + display: block; + } + + .glossary-term { + margin-bottom: 1.5rem; + padding-bottom: 1rem; + border-bottom: 1px solid #eee; + } + + .glossary-term:last-child { + border-bottom: none; + } + + .glossary-term h3 { + color: #667eea; + margin-bottom: 0.5rem; + font-size: 1.1rem; + } + + .glossary-term p { + line-height: 1.7; + color: #555; + } + + .back-link { + margin-bottom: 2rem; + } + @media (max-width: 768px) { .container { padding: 10px; @@ -190,24 +267,34 @@
-

PRAS Reliability Assessment Report

+

Resource Adequacy Report

+ +

Simulation Summary

-
- - -
Samples
-
-
- - -
Events
+
    +
  • Simulation Period: -
    • +
  • Samples: -
    • +
  • System LOLE: -
    • +
  • System NEUE: -
    • +
  • Events: -
    • +
  • Event threshold: -
    • +
+
+
+ Glossary of Terms +
+
+ Generated with PRAS +
- Reliability Events + RA Events - System level summary
@@ -233,6 +320,43 @@

Simulation Summary

+
+ + +
+
+ ← Back to Report Homepage +
+ +
+

Glossary of Terms

+ +
+

Expected Unserved Energy (EUE)

+

Expected Unserved Energy (EUE) is the expected (average) total energy shortfall over the study period. It may be expressed in energy units (e.g. GWh per year).

+
+ +
+

Loss-of-Load Expectation (LOLE)

+

Loss-of-Load Expectation (LOLE) is the expected (average) count of periods experiencing shortfall over the study period. It is expressed in terms of event-periods (e.g. event-hours per year, event-days per year). When reported in terms of event-hours, LOLE is sometimes referred to as LOLH (loss-of-load hours).

+
+ +
+

Normalized Expected Unserved Energy (NEUE)

+

Normalized Expected Unserved Energy (NEUE) is the Expected Unserved Energy normalized against the system's total energy demand and expressed as a fraction, typically shown as a percentage or in parts-per-million (ppm).

+
+ +
+

Resource Adequacy (RA) Event

+

A Resource Adequacy event in this report is a set of contiguous timeperiods where the system EUE in those timeperiods exceed the event threshold.

+
+ +
+

Event Threshold

+

The minimum energy shortfall required for a time period to be considered part of a resource adequacy event.

+
+
+