Overhaul reference comparison: new error metrics, DataFrames CSV, richer HTML table

.github/workflows/msl-test.yml

@@ -145,7 +145,11 @@ jobs:
           fi
 
       - name: Copy new results into gh-pages tree
-        run: rsync -a results/ site/results/
+        env:
+          BM_VERSION: ${{ steps.versions.outputs.bm_version }}
+        run: |
+          rm -rf "site/results/${BM_VERSION}/${LIB_NAME}/${LIB_VERSION}"
+          rsync -a results/ site/results/
 
       - name: Generate landing page
         run: python3 .github/scripts/gen_landing_page.py site

Project.toml

@@ -5,6 +5,8 @@ authors = ["AnHeuermann"]
 
 [deps]
 BaseModelica = "a17d5099-185d-4ff5-b5d3-51aa4569e56d"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"

src/BaseModelicaLibraryTesting.jl

@@ -4,9 +4,9 @@ import Pkg
 import OMJulia
 import OMJulia: sendExpression
 import BaseModelica
-import DifferentialEquations: solve, Rodas5P, ReturnCode
+import DifferentialEquations
 import ModelingToolkit
-import Dates: now
+import Dates
 import Printf: @sprintf
 
 include("types.jl")
@@ -22,10 +22,7 @@ include("pipeline.jl")
 
 # Shared types and constants
 export ModelResult, CompareSettings, RunInfo
-export LIBRARY, LIBRARY_VERSION, CMP_REL_TOL, CMP_ABS_TOL
-
-# Comparison configuration
-export configure_comparison!, compare_settings
+export CMP_REL_TOL, CMP_ABS_TOL
 
 # Pipeline phases
 export run_export       # Phase 1: Base Modelica export via OMC

src/pipeline.jl

@@ -57,8 +57,12 @@ end
 
 Run the four-phase pipeline for a single model and return its result.
 """
-function test_model(omc::OMJulia.OMCSession, model::String, results_root::String,
-                    ref_root::String; csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::ModelResult
+function test_model(omc::OMJulia.OMCSession,
+                    model::String,
+                    results_root::String,
+                    ref_root::String;
+                    csv_max_size_mb::Int          = CSV_MAX_SIZE_MB,
+                    settings::CompareSettings     = CompareSettings())::ModelResult
     model_dir = joinpath(results_root, "files", model)
     mkpath(model_dir)
 
@@ -69,27 +73,45 @@ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String
     # Phase 1 ──────────────────────────────────────────────────────────────────
     exp_ok, exp_t, exp_err = run_export(omc, model, model_dir, bm_path)
     exp_ok || return ModelResult(
-        model, false, exp_t, exp_err, false, 0.0, "", false, 0.0, "", 0, 0, 0, "")
+        model, false, exp_t, exp_err, false, 0.0, "", false, 0.0, "", 0, 0, 0, 0.0, "")
 
     # Phase 2 ──────────────────────────────────────────────────────────────────
     par_ok, par_t, par_err, ode_prob = run_parse(bm_path, model_dir, model)
     par_ok || return ModelResult(
-        model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, "")
+        model, true, exp_t, exp_err, false, par_t, par_err, false, 0.0, "", 0, 0, 0, 0.0, "")
+
+    # Resolve reference CSV and comparison signals early so phase 3 can filter
+    # the CSV output to only the signals that will actually be verified.
+    ref_csv     = isempty(ref_root) ? nothing : _ref_csv_path(ref_root, model)
+    cmp_signals = if ref_csv !== nothing
+        sig_file = joinpath(dirname(ref_csv), "comparisonSignals.txt")
+        if isfile(sig_file)
+            String.(filter(s -> lowercase(s) != "time" && !isempty(s), strip.(readlines(sig_file))))
+        else
+            _, ref_data = _read_ref_csv(ref_csv)
+            filter(k -> lowercase(k) != "time", collect(keys(ref_data)))
+        end
+    else
+        String[]
+    end
 
     # Phase 3 ──────────────────────────────────────────────────────────────────
-    sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model; csv_max_size_mb)
+    sim_ok, sim_t, sim_err, sol = run_simulate(ode_prob, model_dir, model;
+                                               csv_max_size_mb, cmp_signals)
 
     # Phase 4 (optional) ───────────────────────────────────────────────────────
     cmp_total, cmp_pass, cmp_skip, cmp_csv = 0, 0, 0, ""
-    if sim_ok && !isempty(ref_root)
-        ref_csv = _ref_csv_path(ref_root, model)
-        if ref_csv !== nothing
-            try
-                cmp_total, cmp_pass, cmp_skip, cmp_csv =
-                    compare_with_reference(sol, ref_csv, model_dir, model)
-            catch e
-                @warn "Reference comparison failed for $model: $(sprint(showerror, e))"
-            end
+    cmp_t = 0.0
+    if sim_ok && ref_csv !== nothing
+        try
+            t0_cmp = time()
+            cmp_total, cmp_pass, cmp_skip, cmp_csv =
+                compare_with_reference(sol, ref_csv, model_dir, model;
+                                       settings, signals = cmp_signals)
+            cmp_t = time() - t0_cmp
+        catch e
+            cmp_t = time() - t0_cmp
+            @warn "Reference comparison failed for $model: $(sprint(showerror, e))"
         end
     end
 
@@ -98,7 +120,7 @@ function test_model(omc::OMJulia.OMCSession, model::String, results_root::String
         true,   exp_t, exp_err,
         true,   par_t, par_err,
         sim_ok, sim_t, sim_err,
-        cmp_total, cmp_pass, cmp_skip, cmp_csv)
+        cmp_total, cmp_pass, cmp_skip, cmp_t, cmp_csv)
 end
 
 # ── Main ───────────────────────────────────────────────────────────────────────
@@ -111,14 +133,15 @@ Discovers models via OMC, runs `test_model` for each, then writes the HTML
 report.  Returns a `Vector{ModelResult}`.
 """
 function main(;
-    library          :: String                = LIBRARY,
-    version          :: String                = LIBRARY_VERSION,
+    library          :: String,
+    version          :: String,
     filter           :: Union{String,Nothing} = nothing,
     omc_exe          :: String                = get(ENV, "OMC_EXE", "omc"),
     results_root     :: String                = "",
     ref_root         :: String                = get(ENV, "MAPLIB_REF", ""),
     bm_options       :: String                = get(ENV, "BM_OPTIONS", "scalarize,moveBindings,inlineFunctions"),
     csv_max_size_mb  :: Int                   = CSV_MAX_SIZE_MB,
+    settings         :: CompareSettings       = CompareSettings(),
 )
     t0 = time()
 
@@ -186,7 +209,7 @@ function main(;
 
         for (i, model) in enumerate(models)
             @info "[$i/$(length(models))] $model"
-            result = test_model(omc, model, results_root, ref_root; csv_max_size_mb)
+            result = test_model(omc, model, results_root, ref_root; csv_max_size_mb, settings)
             push!(results, result)
 
             phase = if result.sim_success && result.cmp_total > 0
@@ -202,7 +225,9 @@ function main(;
             end
             cmp_info = if result.cmp_total > 0
                 skip_note = result.cmp_skip > 0 ? " skip=$(result.cmp_skip)" : ""
-                "  cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note"
+                "  cmp=$(result.cmp_pass)/$(result.cmp_total)$skip_note ($(round(result.cmp_time;digits=2))s)"
+            elseif result.cmp_time > 0
+                "  cmp=n/a ($(round(result.cmp_time;digits=2))s)"
             else
                 ""
             end

src/report.jl

@@ -1,6 +1,6 @@
 # ── HTML report generation ─────────────────────────────────────────────────────
 
-import Dates: now
+import Dates
 import Printf: @sprintf
 
 function _status_cell(ok::Bool, t::Float64, logFile::Union{String,Nothing})
@@ -145,7 +145,7 @@ function generate_report(results::Vector{ModelResult}, results_root::String,
 </head>
 <body>
 <h1>$(info.library) $(info.lib_version) — Base Modelica / MTK Pipeline Test Results</h1>
-<p>Generated: $(now())<br>
+<p>Generated: $(Dates.now())<br>
 OpenModelica: $(info.omc_version)<br>
 OMC options: <code>$(info.omc_options)</code><br>
 BaseModelica.jl: $(basemodelica_jl_version)<br>

src/simulate.jl

@@ -1,6 +1,6 @@
 # ── Phase 3: ODE simulation with DifferentialEquations / MTK ──────────────────
 
-import DifferentialEquations: solve, Rodas5P, ReturnCode
+import DifferentialEquations
 import Logging
 import ModelingToolkit
 import Printf: @sprintf
@@ -9,15 +9,17 @@ import Printf: @sprintf
     run_simulate(ode_prob, model_dir, model; csv_max_size_mb) → (success, time, error, sol)
 
 Solve `ode_prob` with Rodas5P (stiff solver).  On success, also writes the
-full solution as a CSV file `<Short>_sim.csv` in `model_dir`.
+solution as a CSV file `<Short>_sim.csv` in `model_dir`.
 Writes a `<model>_sim.log` file in `model_dir`.
 Returns `nothing` as the fourth element on failure.
 
 CSV files larger than `csv_max_size_mb` MiB are deleted and replaced with a
 `<Short>_sim.csv.toobig` marker so that the report can note the omission.
 """
-function run_simulate(ode_prob, model_dir::String, model::String;
-                      csv_max_size_mb::Int = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any}
+function run_simulate(ode_prob, model_dir::String,
+                      model::String;
+                      cmp_signals    ::Vector{String} = String[],
+                      csv_max_size_mb::Int            = CSV_MAX_SIZE_MB)::Tuple{Bool,Float64,String,Any}
     sim_success = false
     sim_time    = 0.0
     sim_error   = ""
@@ -27,16 +29,20 @@ function run_simulate(ode_prob, model_dir::String, model::String;
     println(log_file, "Model:   $model")
     logger = Logging.SimpleLogger(log_file, Logging.Debug)
     t0 = time()
+
+    # Read interval before overwriting it
+    interval = something(get(ode_prob.kwargs, :saveat, nothing),
+                         (ode_prob.tspan[end] - ode_prob.tspan[1]) / 500)
+
     try
         # Rodas5P handles stiff DAE-like systems well.
         # Redirect all library log output (including Symbolics/MTK warnings)
         # to the log file so they don't clutter stdout.
         sol = Logging.with_logger(logger) do
-            # Overwrite saveat, always use dense output.
-            solve(ode_prob, Rodas5P(); saveat = Float64[], dense = true)
+            DifferentialEquations.solve(ode_prob, DifferentialEquations.Rodas5P(); saveat = Float64[], dense = true)
         end
         sim_time = time() - t0
-        if sol.retcode == ReturnCode.Success
+        if sol.retcode == DifferentialEquations.ReturnCode.Success
             sys    = sol.prob.f.sys
             n_vars = length(ModelingToolkit.unknowns(sys))
             n_obs  = length(ModelingToolkit.observed(sys))
@@ -67,20 +73,28 @@ function run_simulate(ode_prob, model_dir::String, model::String;
             sys      = sol.prob.f.sys
             vars     = ModelingToolkit.unknowns(sys)
             obs_eqs  = ModelingToolkit.observed(sys)
-            obs_syms = [eq.lhs for eq in obs_eqs]
+            # Only save observed variables that appear in cmp_signals.
+            # This avoids writing thousands of algebraic variables to disk when
+            # only a handful are actually verified during comparison.
+            norm_cmp = Set(_normalize_var(s) for s in cmp_signals)
+            obs_eqs_filtered = isempty(norm_cmp) ? obs_eqs :
+                filter(eq -> _normalize_var(string(eq.lhs)) in norm_cmp, obs_eqs)
+            obs_syms = [eq.lhs for eq in obs_eqs_filtered]
             col_names = vcat(
                 [_clean_var_name(string(v)) for v in vars],
                 [_clean_var_name(string(s)) for s in obs_syms],
             )
             open(sim_csv, "w") do f
                 println(f, join(["time"; col_names], ","))
-                for (ti, t) in enumerate(sol.t)
+                t_csv = range(ode_prob.tspan[1], ode_prob.tspan[end]; step = interval)
+                for t in t_csv
                     row = [@sprintf("%.10g", t)]
+                    u   = sol(Float64(t))
                     for vi in eachindex(vars)
-                        push!(row, @sprintf("%.10g", sol[vi, ti]))
+                        push!(row, @sprintf("%.10g", u[vi]))
                     end
                     for sym in obs_syms
-                        val = try Float64(sol(t; idxs = sym)) catch; NaN end
+                        val = try Float64(sol(Float64(t); idxs = sym)) catch; NaN end
                         push!(row, @sprintf("%.10g", val))
                     end
                     println(f, join(row, ","))
@@ -90,7 +104,6 @@ function run_simulate(ode_prob, model_dir::String, model::String;
             if csv_bytes > csv_max_size_mb * 1024^2
                 csv_mb = round(csv_bytes / 1024^2; digits=1)
                 @warn "Simulation CSV for $model is $(csv_mb) MB (> $(csv_max_size_mb) MB limit); skipping."
-                rm(sim_csv)
                 write(sim_csv * ".toobig", string(csv_bytes))
             end
         catch e

src/types.jl

@@ -1,9 +1,6 @@
 # ── Shared constants ───────────────────────────────────────────────────────────
 
-const LIBRARY         = "Modelica"
-const LIBRARY_VERSION = "4.1.0"
-
-# Comparison tolerances (2 % relative, 1e-6 absolute — matches Modelica
+# Comparison tolerances (2 % relative, 1e-4 absolute — matches Modelica
 # Association compliance tooling defaults).
 const CMP_REL_TOL = 0.02
 const CMP_ABS_TOL = 1e-6
@@ -20,21 +17,12 @@ const CSV_MAX_SIZE_MB = 20
 Mutable configuration struct for signal comparison.
 
 # Fields
-- `rel_tol`  — maximum allowed relative error (default: `$(CMP_REL_TOL)`, i.e. 2 %).
-- `abs_tol`  — hard absolute-error floor used when signals are near zero
-               (default: `$(CMP_ABS_TOL)`).
-- `error_fn` — selects the point-wise pass/fail function.  One of:
-  - `:mixed`    — scale-aware relative error (default, recommended);
-  - `:relative` — classic relative error (may reject valid zero-crossing signals);
-  - `:absolute` — pure absolute error.
-
-Use `configure_comparison!` to update the module-level defaults, or construct a
-local instance to pass to `compare_with_reference` for a single run.
+- `abs_tol`  — hard absolute-error (default: `nothing`)
+- `rel_tol`  — maximum allowed globally scaled relative error (default: `$(CMP_REL_TOL)`, i.e. 2 %).
 """
 Base.@kwdef mutable struct CompareSettings
+    abs_tol  :: Union{Float64,Nothing} = nothing
     rel_tol  :: Float64 = CMP_REL_TOL
-    abs_tol  :: Float64 = CMP_ABS_TOL
-    error_fn :: Symbol  = :mixed
 end
 
 # ── Run metadata ───────────────────────────────────────────────────────────────
@@ -94,5 +82,6 @@ struct ModelResult
     cmp_total      :: Int     # signals actually compared (found in simulation)
     cmp_pass       :: Int
     cmp_skip       :: Int     # reference signals not found in simulation
+    cmp_time       :: Float64 # wall time for comparison phase (0.0 if skipped)
     cmp_csv        :: String  # absolute path to diff CSV; "" if all pass or no comparison
 end