From c1b4399b392c9cbc4a5f787e146b84b2656fe028 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:07:45 +0200 Subject: [PATCH 001/145] test(qa): a NaN single-objective value is logged as [nan] where Optuna logs nan Found by diffing the INFO/WARNING log lines of a real-world-shaped run (completed, pruned, failed-by-exception, NaN and multi-objective trials) against Optuna 5.0.0. Every line matched except two: the exception type name (InvalidOperationException vs RuntimeError - language, expected) and the NaN failure line. optuna/study/_optimize.py::_log_failed_trial warns f"Trial {trial_number} failed with value {repr(value_or_values)}." with the objective's own return value - a float for a single-objective study, so "Trial 0 failed with value nan.". The single-objective Study.Optimize overload wraps the scalar in a one-element array before the loop, and Internal/Optimize.cs::LogFailedTrial prints it with ListRepr: "[nan]". Same scalar-wrapping root cause as QA-12 (the skipped-tell message), different message. New [OpenBugs] test Optimize_NaNScalarObjective_LogsScalarValueLikePython in test/OptunaSharp.Tests/QA/RealWorldExampleBugTests.cs (fails today). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../QA/RealWorldExampleBugTests.cs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/OptunaSharp.Tests/QA/RealWorldExampleBugTests.cs b/test/OptunaSharp.Tests/QA/RealWorldExampleBugTests.cs index c7126ad..0275643 100644 --- a/test/OptunaSharp.Tests/QA/RealWorldExampleBugTests.cs +++ b/test/OptunaSharp.Tests/QA/RealWorldExampleBugTests.cs @@ -37,6 +37,60 @@ private static string CaptureStderr(Action action) return writer.ToString(); } + /// Log handler that keeps the text of every record it receives, in order. + /// Registered for one test at a time; records from other threads' studies can interleave. + private sealed class CollectingHandler : OptunaSharp.Logging.ILogHandler + { + /// The messages received so far, oldest first. + public System.Collections.Generic.List Messages { get; } = new(); + + /// Stores the record's message; never throws, so it cannot disturb the code under test. + /// The log record delivered by . + public void Emit(in OptunaSharp.Logging.LogRecord record) + { + lock (Messages) + Messages.Add(record.Message); + } + } + + /// + /// A single-objective objective that returns NaN must be logged with the value it returned, as + /// Optuna's repr(value_or_values) does: Trial 0 failed with value nan.; the port logs + /// [nan]. + /// + /// + /// optuna/study/_optimize.py (_log_failed_trial) warns + /// f"Trial {trial_number} failed with value {repr(value_or_values)}." with the objective's own + /// return value — a float for a single-objective study. The single-objective Study.Optimize + /// overload wraps the scalar into a one-element array before the loop runs, and + /// Internal/Optimize.cs (LogFailedTrial) prints that array with ListRepr. The same + /// scalar-wrapping root cause as QA-12 (the skipped-tell message), on a different message. Found by + /// diffing the logs of a real-world run (examples/0_GpuFineTunePlanner.cs returns NaN for + /// diverged fp16 runs) against Optuna 5.0.0. + /// + [TestMethod] + [OpenBugs] + public void Optimize_NaNScalarObjective_LogsScalarValueLikePython() + { + var handler = new CollectingHandler(); + int verbosity = OptunaSharp.Logging.OptunaLogging.GetVerbosity(); + OptunaSharp.Logging.OptunaLogging.SetVerbosity(OptunaSharp.Logging.LogLevel.Info); + OptunaSharp.Logging.OptunaLogging.AddHandler(handler); + try + { + var study = Optuna.CreateStudy(); + study.Optimize(_ => double.NaN, nTrials: 1); + } + finally + { + // Global logging state: always restore it, or later tests log at the wrong level. + OptunaSharp.Logging.OptunaLogging.RemoveHandler(handler); + OptunaSharp.Logging.OptunaLogging.SetVerbosity(verbosity); + } + + handler.Messages.Should().Contain("Trial 0 failed with value nan."); + } + /// /// The progress bar's running best must be formatted like Optuna's f"{study.best_value:.6g}" /// (six significant digits: 0.930382, 1.23457e+06); the port prints the full float repr. From 94de6f97cbc22e14f933b9c3338083e5d40b7c66 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:11:14 +0200 Subject: [PATCH 002/145] feat(examples): 0_ContainerLoadingStopAtBound - enqueue_trial(skip_if_exists) + stop() at a provable bound A tunable bin-packing heuristic (size^p ordering with a seeded tie-break jitter, first/best/worst fit, repair passes that try to empty the least-full trailer) loads Falkenauer-style "triplet" shipments into 13.6 m trailers. The load planners' rules of thumb (first-fit and best-fit decreasing) are enqueued twice each with skip_if_exists=True (the duplicates are skipped), and a callback calls study.Stop() once a plan reaches the lower bound ceil(sum/capacity). Here FFD and BFD need 46 trailers, the tuned heuristic 45, the bound is 40 - so the budget is used up and the output says so. Checked while writing it (not bugs, faithful to Optuna): when stop() fires right after the first trial, the second enqueued trial stays WAITING with no params and study.trials includes it, as Python's does; the example summarizes COMPLETE trials only and explains why. Also checked in this round (no defect): sampler-fallback and enqueue warnings match Optuna verbatim (ExperimentalWarning aside - removed on purpose earlier); multi-objective trials_dataframe columns (flat and multi_index) match pandas; 3,000 TPE trials take 11.6 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- examples/0_ContainerLoadingStopAtBound.cs | 164 ++++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 165 insertions(+) create mode 100644 examples/0_ContainerLoadingStopAtBound.cs diff --git a/examples/0_ContainerLoadingStopAtBound.cs b/examples/0_ContainerLoadingStopAtBound.cs new file mode 100644 index 0000000..644a139 --- /dev/null +++ b/examples/0_ContainerLoadingStopAtBound.cs @@ -0,0 +1,164 @@ +#:project ../src/OptunaSharp.Core/OptunaSharp.Core.csproj +#:property PublishAot=false + +// Container loading — tune a bin-packing heuristic that assigns a week's shipments to containers, starting +// from the load planners' rules of thumb (enqueued with `skip_if_exists`, so re-running never duplicates +// them) and stopping the study the moment a plan reaches the provable lower bound (`study.stop()` from a +// callback): there is nothing better to find, so the remaining budget is not spent. +// +// source: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.enqueue_trial +// (`enqueue_trial(params, skip_if_exists=True)`) +// https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.stop +// https://en.wikipedia.org/wiki/Bin_packing_problem (first-fit / best-fit decreasing, the L1 bound) +// +// Real objective, a real heuristic: 120 shipments (pallet-metres) generated as Falkenauer-style triplets — +// every trailer of the known optimum holds exactly three of them — are packed into 13.6 m trailers. The heuristic orders the shipments by size^p (p tunable; +// with noise of a tunable seed to break ties differently), places each by first-, best- or worst-fit, then +// runs a tunable number of repair passes that try to empty the least-full trailer into the others. The +// score is the number of trailers plus a small term for how empty the emptiest one is (an emptier last +// trailer is easier to cancel). The lower bound ⌈Σ size / capacity⌉ is known, so reaching it ends the study. +// +// Run: dotnet run examples/0_ContainerLoadingStopAtBound.cs + +using System.Globalization; +using OptunaSharp; +using OptunaSharp.Collections; +using OptunaSharp.Logging; +using OptunaSharp.Samplers; + +OptunaLogging.SetVerbosity(LogLevel.Warning); // silence the per-trial INFO lines + +var shipments = Shipments.Week(count: 120, seed: 37); +const double Capacity = 13.6; +int lowerBound = (int)Math.Ceiling(shipments.Sum() / Capacity); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0} shipments, {1:F1} pallet-metres; lower bound: {2} trailers", shipments.Length, shipments.Sum(), lowerBound)); + +var study = Optuna.CreateStudy(sampler: new TPESampler(seed: 38, nStartupTrials: 15), studyName: "trailer-loading"); +// the planners' rules of thumb: first-fit-decreasing and best-fit-decreasing, no repair. +foreach (string fit in new[] { "first", "best" }) + for (int twice = 0; twice < 2; twice++) // the second enqueue is skipped: already queued + study.EnqueueTrial(new OrderedDict { ["order_power"] = 1.0, ["noise_seed"] = 0L, ["fit"] = fit, ["repair_passes"] = 0L }, + skipIfExists: true); + +void StopAtBound(Study s, FrozenTrial t) +{ + if (t.Value is double v && Math.Floor(v) <= lowerBound) + s.Stop(); // provably optimal trailer count: done +} +study.Optimize(t => Objective(t, shipments, Capacity), nTrials: 300, callbacks: new Action[] { StopAtBound }); + +// note: if the study stops early, a still-queued enqueued trial stays WAITING in the storage (it would run +// first on the next optimize) — so summarize only the COMPLETE trials. +var completed = study.Trials.Where(t => t.State == TrialState.Complete).ToList(); +var rulesOfThumb = completed.Where(t => (long)t.Params["repair_passes"]! == 0 && (double)t.Params["order_power"]! == 1.0 && (long)t.Params["noise_seed"]! == 0).Take(2).Select(t => $"{t.Params["fit"]}-fit decreasing → {Math.Floor(t.Value!.Value)} trailers").ToList(); +Console.WriteLine("planners' heuristics: " + string.Join("; ", rulesOfThumb)); +var best = study.BestTrial; +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "tuned: {0} trailers after {1} trials ({2}); emptiest trailer {3:P0} full", + Math.Floor(best.Value!.Value), completed.Count, Math.Floor(best.Value.Value) <= lowerBound ? "hit the lower bound — study stopped" : "budget used up", + best.UserAttrs["emptiest_fill"])); +foreach (var kv in best.Params) + Console.WriteLine($" {kv.Key,-14} {kv.Value}"); + +/// Packs the week's shipments with the trial's heuristic settings. +/// The trial proposing the ordering power, tie-break seed, fit rule and repair effort. +/// Shipment sizes in pallet-metres. +/// Trailer capacity in pallet-metres. +/// Trailers used plus (0.99 × the emptiest trailer's fill), so fewer and emptier-last plans win. +static double Objective(Trial t, double[] items, double capacity) +{ + double power = t.SuggestFloat("order_power", 0.25, 4.0); // sort key: size^p (p = 1: plain decreasing) + long noiseSeed = t.SuggestInt("noise_seed", 0, 50); // tie-breaking perturbation of the order + string fit = t.SuggestCategorical("fit", new[] { "first", "best", "worst" }); + long passes = t.SuggestInt("repair_passes", 0, 20); // try to empty the least-full trailer + var bins = Packer.Pack(items, capacity, power, (int)noiseSeed, fit, (int)passes); + double emptiest = bins.Min(b => b.Sum()) / capacity; + t.SetUserAttr("emptiest_fill", emptiest); + return bins.Count + 0.99 * emptiest; +} + +/// The week's shipments. +static class Shipments +{ + /// + /// Generates a "triplet" week (Falkenauer's hard class): the shipments come in groups of three that + /// exactly fill one trailer, so the lower bound is achievable — but only by packing the right triples + /// together, which decreasing-order heuristics rarely do. + /// + /// Number of shipments (a multiple of three). + /// Data seed. + /// Sizes in pallet-metres, shuffled. + public static double[] Week(int count, int seed) + { + const double Capacity = 13.6; + var rng = new Random(seed); // data generation only — samplers never use System.Random + var items = new List(); + for (int k = 0; k < count / 3; k++) + { + // three loads of roughly a quarter to a half trailer that sum to exactly one trailer (0.1 m grid). + int total = (int)Math.Round(Capacity * 10), a = 34 + rng.Next(34), b = 34 + rng.Next(Math.Max(1, total - a - 34 - 34)); + items.AddRange(new[] { a / 10.0, b / 10.0, (total - a - b) / 10.0 }); + } + var arr = items.ToArray(); + rng.Shuffle(arr); + return arr; + } +} + +/// A parameterized decreasing-order bin-packing heuristic with a repair step. +static class Packer +{ + /// Packs items into bins. + /// Item sizes. + /// Bin capacity. + /// Exponent of the ordering key. + /// Seed of the small multiplicative jitter applied to the key. + /// "first", "best" or "worst" fit. + /// Repair passes. + /// The bins, each a list of item sizes. + public static List> Pack(double[] items, double capacity, double power, int noiseSeed, string fit, int passes) + { + var rng = new Random(noiseSeed); + var order = items.Select(s => (Size: s, Key: Math.Pow(s, power) * (1 + 0.05 * rng.NextDouble()))).OrderByDescending(p => p.Key).Select(p => p.Size); + var bins = new List>(); + var loads = new List(); + foreach (double s in order) + Place(s); + for (int p = 0; p < passes && bins.Count > 1; p++) + { + int victim = loads.IndexOf(loads.Min()); + var moved = bins[victim]; + bins.RemoveAt(victim); + loads.RemoveAt(victim); + int before = bins.Count; + foreach (double s in moved.OrderByDescending(x => x)) + Place(s); + if (bins.Count > before) + break; // could not empty it: stop repairing + } + return bins; + + // put one item into an existing bin by the fit rule, or open a new bin. + void Place(double s) + { + int chosen = -1; + for (int b = 0; b < bins.Count; b++) + { + if (loads[b] + s > capacity + 1e-9) + continue; + // best fit = the fullest bin that still fits; worst fit = the emptiest one. + if (chosen < 0 || (fit == "best" && loads[b] > loads[chosen]) || (fit == "worst" && loads[b] < loads[chosen])) + chosen = b; + if (fit == "first") + break; // first fit: the first bin that fits wins + } + if (chosen < 0) + { + bins.Add(new List()); + loads.Add(0); + chosen = bins.Count - 1; + } + bins[chosen].Add(s); + loads[chosen] += s; + } + } +} diff --git a/examples/README.md b/examples/README.md index 01eadd6..83a752d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -58,6 +58,7 @@ classes). | [`0_RateLimiterHostedService.cs`](0_RateLimiterHostedService.cs) | Generic Host + DI (`AddOptunaStudyFactory` …), cancellation | a `BackgroundService` re-tunes an API gateway's token bucket every telemetry window | | [`0_PressureVesselParetoReport.cs`](0_PressureVesselParetoReport.cs) | NSGA-II, mixed directions, constraints, Pareto + hypervolume charts | pressure-vessel cost vs. capacity under the ASME thickness rules (RE23) | | [`0_SpamFilterTimeBudget.cs`](0_SpamFilterTimeBudget.cs) | TPE with only a `timeout`, `show_progress_bar` | a naive-Bayes spam filter tuned inside a fixed ten-second slot | +| [`0_ContainerLoadingStopAtBound.cs`](0_ContainerLoadingStopAtBound.cs) | `enqueue_trial(skip_if_exists=True)`, `study.stop()` from a callback | a bin-packing heuristic for trailer loading, warm-started from planners' rules, stopped at the lower bound | Notes from running them: From acd2162cc5c7e8aac9571b11c9e9e380ba72973d Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:13:34 +0200 Subject: [PATCH 003/145] feat(examples): 0_DistributedJournalHyperband - 3 processes x n_jobs=2 on one journal file Optuna's shared-file-system recipe: the parent creates the study in a JournalFileBackend file and starts three worker processes (the same program with --worker), each joining via load_study and running 16 trials on two threads with HyperbandPruner; then it audits the file. Objective: mini-batch SGD ridge regression (optional pairwise-product features) on a deterministic house-price set, validation RMSE reported every epoch so Hyperband's brackets compare trials from different processes at the same epoch. Run here: 48 trials, numbers 0..47 each exactly once, 34 pruned / 14 complete, 546 of 1,296 epochs trained (58 % saved), best RMSE 0.098 at the 0.1 noise floor (mean predictor 0.676). No defect found. Co-Authored-By: Claude Opus 5.5 (1M context) --- examples/0_DistributedJournalHyperband.cs | 167 ++++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 168 insertions(+) create mode 100644 examples/0_DistributedJournalHyperband.cs diff --git a/examples/0_DistributedJournalHyperband.cs b/examples/0_DistributedJournalHyperband.cs new file mode 100644 index 0000000..ce34906 --- /dev/null +++ b/examples/0_DistributedJournalHyperband.cs @@ -0,0 +1,167 @@ +#:project ../src/OptunaSharp.Core/OptunaSharp.Core.csproj +#:property PublishAot=false + +// Distributed tuning on a shared journal file — three worker processes, each running two trial threads +// (`n_jobs=2`), optimize one study through a single JournalFileBackend file (Optuna's recipe for clusters +// that only share a file system), and HyperbandPruner coordinates early stopping across all six workers +// because every intermediate value lands in the same journal. +// +// source: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.storages.journal.JournalFileBackend.html +// (the append-only, lock-protected journal file several processes can share) +// https://optuna.readthedocs.io/en/stable/tutorial/10_key_features/004_distributed.html +// https://optuna.readthedocs.io/en/stable/reference/generated/optuna.pruners.HyperbandPruner.html +// +// Real objective, real training: a deterministic house-price regression set (8 features with interactions, +// 3,000 train / 1,000 validation rows) and a polynomial-feature ridge model fitted by mini-batch SGD; every +// epoch reports the validation RMSE, so Hyperband's brackets can compare trials from different processes at +// the same epoch. The parent creates the study, starts the workers, waits, and then audits the shared file: +// unique trial numbers, every trial finished, and how much training the pruner saved. +// +// Run: dotnet run examples/0_DistributedJournalHyperband.cs + +using System.Diagnostics; +using System.Globalization; +using OptunaSharp; +using OptunaSharp.Logging; +using OptunaSharp.Pruners; +using OptunaSharp.Samplers; +using OptunaSharp.Storages.Journal; + +const string StudyName = "house-prices-sgd"; +const int MaxEpochs = 27; +string journal = args.Length > 1 ? args[1] : Path.Combine(Path.GetTempPath(), "optunasharp-house-prices.journal"); + +if (args.Length > 0 && args[0] == "--worker") +{ + // child mode: join the study through the shared file and run 16 trials on two threads. + OptunaLogging.SetVerbosity(LogLevel.Warning); + var data = Houses.Generate(seed: 41); + var worker = Optuna.LoadStudy(StudyName, new JournalStorage(new JournalFileBackend(journal)), + sampler: new TPESampler(seed: int.Parse(args[2], CultureInfo.InvariantCulture), nStartupTrials: 8), + pruner: new HyperbandPruner(minResource: 1, maxResource: MaxEpochs, reductionFactor: 3)); + worker.Optimize(t => Objective(t, data), nTrials: 16, nJobs: 2); + return; +} + +OptunaLogging.SetVerbosity(LogLevel.Warning); +File.Delete(journal); +Optuna.CreateStudy(storage: new JournalStorage(new JournalFileBackend(journal)), studyName: StudyName, direction: StudyDirection.Minimize); + +var sw = Stopwatch.StartNew(); +string dll = typeof(Houses).Assembly.Location; +var workers = Enumerable.Range(1, 3).Select(id => +{ + var psi = new ProcessStartInfo("dotnet") { RedirectStandardError = true, RedirectStandardOutput = true }; + foreach (var a in new[] { "exec", dll, "--worker", journal, id.ToString(CultureInfo.InvariantCulture) }) + psi.ArgumentList.Add(a); + return Process.Start(psi)!; +}).ToList(); +var errors = workers.Select(w => { w.StandardOutput.ReadToEnd(); string e = w.StandardError.ReadToEnd(); w.WaitForExit(); return (w.ExitCode, e); }).ToList(); +sw.Stop(); + +var study = Optuna.LoadStudy(StudyName, new JournalStorage(new JournalFileBackend(journal))); +var trials = study.Trials; +var numbers = trials.Select(t => t.Number).OrderBy(n => n).ToList(); +int epochs = trials.Sum(t => t.IntermediateValues.Count); +Console.WriteLine($"3 processes × 2 threads finished in {sw.Elapsed.TotalSeconds:F1} s, exit codes: {string.Join(", ", errors.Select(e => e.ExitCode))}"); +foreach (var (code, err) in errors.Where(e => e.ExitCode != 0)) + Console.WriteLine("worker stderr: " + err.Trim().Split('\n').Last()); +Console.WriteLine($"trials in the journal: {trials.Count}; numbers 0..{trials.Count - 1} each exactly once: {numbers.SequenceEqual(Enumerable.Range(0, trials.Count))}"); +Console.WriteLine("states: " + string.Join(", ", trials.GroupBy(t => t.State).Select(g => $"{g.Key} {g.Count()}"))); +Console.WriteLine($"epochs trained: {epochs} of {trials.Count * MaxEpochs} without pruning ({1 - (double)epochs / (trials.Count * MaxEpochs):P0} saved)"); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "best validation RMSE: {0:F4} (trial #{1}); predicting the mean: {2:F4}", + study.BestValue, study.BestTrial.Number, Houses.Generate(seed: 41).MeanBaselineRmse())); +foreach (var kv in study.BestParams) + Console.WriteLine($" {kv.Key,-8} {kv.Value}"); + +/// Fits one SGD ridge configuration epoch by epoch, reporting the validation RMSE. +/// The trial proposing the learning rate, its decay, the L2 strength, the batch size and the feature degree. +/// The house-price data. +/// The validation RMSE after the last epoch. +/// Hyperband stopped the configuration at one of its rungs. +static double Objective(Trial t, Houses d) +{ + double lr = t.SuggestFloat("lr", 1e-4, 0.5, log: true); + double decay = t.SuggestFloat("decay", 0.85, 1.0); + double l2 = t.SuggestFloat("l2", 1e-6, 1.0, log: true); + int batch = (int)t.SuggestCategorical("batch", new[] { 8, 32, 128 }); + bool quadratic = t.SuggestCategorical("quadratic", new[] { false, true }); // add pairwise products + var xs = quadratic ? d.TrainQ : d.Train; + var vs = quadratic ? d.ValidQ : d.Valid; + var w = new double[xs[0].Length]; + double b = 0, rmse = double.NaN; + for (int epoch = 1; epoch <= MaxEpochs; epoch++) + { + var order = Enumerable.Range(0, xs.Length).ToArray(); + new Random(epoch).Shuffle(order); + for (int s = 0; s < order.Length; s += batch) + { + var g = new double[w.Length]; + double gb = 0; + int end = Math.Min(s + batch, order.Length); + for (int k = s; k < end; k++) + { + int i = order[k]; + double err = Dot(w, xs[i]) + b - d.TrainY[i]; + for (int j = 0; j < w.Length; j++) g[j] += err * xs[i][j]; + gb += err; + } + for (int j = 0; j < w.Length; j++) w[j] -= lr * (g[j] / (end - s) + l2 * w[j]); + b -= lr * gb / (end - s); + } + lr *= decay; + rmse = Math.Sqrt(Enumerable.Range(0, vs.Length).Average(i => Math.Pow(Dot(w, vs[i]) + b - d.ValidY[i], 2))); + if (!double.IsFinite(rmse)) rmse = 1e3; // diverged: a bad finite value + t.Report(rmse, epoch); + if (t.ShouldPrune()) + throw new TrialPruned(); + } + return rmse; + + static double Dot(double[] w, double[] x) + { + double s = 0; + for (int j = 0; j < w.Length; j++) s += w[j] * x[j]; + return s; + } +} + +/// A standardized house-price regression set, with an optional quadratic feature expansion. +/// Training rows (8 standardized features). +/// Training rows with all pairwise products appended. +/// Training targets (log price, centred). +/// Validation rows. +/// Validation rows with pairwise products. +/// Validation targets. +sealed record Houses(double[][] Train, double[][] TrainQ, double[] TrainY, double[][] Valid, double[][] ValidQ, double[] ValidY) +{ + /// Generates the data; the same seed gives the same houses. + /// Data seed. + /// The data set. + public static Houses Generate(int seed) + { + var rng = new Random(seed); // data generation only — samplers never use System.Random + int n = 4000; + var x = new double[n][]; + var y = new double[n]; + for (int i = 0; i < n; i++) + { + var r = Enumerable.Range(0, 8).Select(_ => Normal(rng)).ToArray(); // area, rooms, age, distance, … + y[i] = 0.5 * r[0] + 0.2 * r[1] - 0.25 * r[2] - 0.3 * r[3] + 0.15 * r[0] * r[4] - 0.1 * r[3] * r[3] + 0.1 * Normal(rng); + x[i] = r; + } + double[] Quad(double[] r) => r.Concat(from a in Enumerable.Range(0, 8) from c in Enumerable.Range(a, 8 - a) select r[a] * r[c]).ToArray(); + double mean = y.Take(3000).Average(); + var yc = y.Select(v => v - mean).ToArray(); + return new Houses(x[..3000], x[..3000].Select(Quad).ToArray(), yc[..3000], x[3000..], x[3000..].Select(Quad).ToArray(), yc[3000..]); + } + + /// RMSE of predicting the training mean on the validation rows. + /// The baseline RMSE. + public double MeanBaselineRmse() => Math.Sqrt(ValidY.Average(v => v * v)); + + /// A standard normal draw (Box–Muller). + /// The random stream. + /// A N(0, 1) variate. + private static double Normal(Random rng) => Math.Sqrt(-2 * Math.Log(1 - rng.NextDouble())) * Math.Cos(2 * Math.PI * rng.NextDouble()); +} diff --git a/examples/README.md b/examples/README.md index 83a752d..df02f9f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -58,6 +58,7 @@ classes). | [`0_RateLimiterHostedService.cs`](0_RateLimiterHostedService.cs) | Generic Host + DI (`AddOptunaStudyFactory` …), cancellation | a `BackgroundService` re-tunes an API gateway's token bucket every telemetry window | | [`0_PressureVesselParetoReport.cs`](0_PressureVesselParetoReport.cs) | NSGA-II, mixed directions, constraints, Pareto + hypervolume charts | pressure-vessel cost vs. capacity under the ASME thickness rules (RE23) | | [`0_SpamFilterTimeBudget.cs`](0_SpamFilterTimeBudget.cs) | TPE with only a `timeout`, `show_progress_bar` | a naive-Bayes spam filter tuned inside a fixed ten-second slot | +| [`0_DistributedJournalHyperband.cs`](0_DistributedJournalHyperband.cs) | 3 processes × `n_jobs=2` on one `JournalFileBackend`, `HyperbandPruner` | SGD ridge house-price model; Hyperband's brackets span all six workers through the shared file | | [`0_ContainerLoadingStopAtBound.cs`](0_ContainerLoadingStopAtBound.cs) | `enqueue_trial(skip_if_exists=True)`, `study.stop()` from a callback | a bin-packing heuristic for trailer loading, warm-started from planners' rules, stopped at the lower bound | Notes from running them: From 5101645ade5306130da0b14a9dcb43fefbd8eaed Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:14:50 +0200 Subject: [PATCH 004/145] feat(examples): 0_ChurnScorecardTerminator - the Terminator's documented default recipe Every trial of an L2 logistic churn scorecard (optional quadratic features, gradient descent) reports its 5 CV fold accuracies with TerminatorFunctions.ReportCrossValidationScores, and the default TerminatorCallback (GP regret bound vs CrossValidationErrorEvaluator) ends the study. The same seeded search is also run without the terminator for comparison: it stopped at trial 20 of 120, and the remaining 100 trials would have added +0.05 percentage points - inside the +-2 point fold noise it reads from terminator:cv_scores. The per-trial `optuna.terminator` FutureWarning is Optuna's own behavior (deprecated in v4.9). No defect found. Co-Authored-By: Claude Opus 5.5 (1M context) --- examples/0_ChurnScorecardTerminator.cs | 130 +++++++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 131 insertions(+) create mode 100644 examples/0_ChurnScorecardTerminator.cs diff --git a/examples/0_ChurnScorecardTerminator.cs b/examples/0_ChurnScorecardTerminator.cs new file mode 100644 index 0000000..6ee8a36 --- /dev/null +++ b/examples/0_ChurnScorecardTerminator.cs @@ -0,0 +1,130 @@ +#:project ../src/OptunaSharp.Core/OptunaSharp.Core.csproj +#:property PublishAot=false + +// When to stop tuning — Optuna's Terminator in its documented default form: every trial reports its +// cross-validation fold scores (`report_cross_validation_scores`), and TerminatorCallback ends the study once +// the GP-estimated room for improvement (the regret bound) falls below the noise of the CV estimate itself — +// past that point, a "better" trial is indistinguishable from a lucky fold split. +// +// source: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.terminator.TerminatorCallback.html +// (TerminatorCallback with the default RegretBoundEvaluator + CrossValidationErrorEvaluator, and +// `report_cross_validation_scores`) +// +// Real objective, real training: a deterministic telecom-churn table (2,000 customers, 10 features with +// interactions) and an L2-regularized logistic regression with optional quadratic features, fitted by +// gradient descent; 5-fold CV accuracy is maximized. The trial budget is 120, but the terminator usually +// stops much earlier — the printout shows where, and how little the best score moved after that point in a +// separate run that ignores the terminator. (Optuna has deprecated `optuna.terminator` — v4.9, removal in +// v6 — so a FutureWarning prints; it is still the canonical recipe in Optuna 5.) +// +// Run: dotnet run examples/0_ChurnScorecardTerminator.cs + +using System.Globalization; +using OptunaSharp; +using OptunaSharp.Logging; +using OptunaSharp.Samplers; +using OptunaSharp.Terminator; + +OptunaLogging.SetVerbosity(LogLevel.Warning); // silence the per-trial INFO lines + +var (x, y) = Customers.Generate(rows: 2000, seed: 43); + +// the run with the terminator. +var stopped = Optuna.CreateStudy(sampler: new TPESampler(seed: 44), direction: StudyDirection.Maximize, studyName: "churn-terminated"); +stopped.Optimize(t => Objective(t, x, y), nTrials: 120, callbacks: new Action[] { new TerminatorCallback() }); + +// the same search without it, to see what the remaining budget would have bought. +var full = Optuna.CreateStudy(sampler: new TPESampler(seed: 44), direction: StudyDirection.Maximize, studyName: "churn-full"); +full.Optimize(t => Objective(t, x, y), nTrials: 120); + +int stopAt = stopped.Trials.Count; +double bestAtStop = full.Trials.Take(stopAt).Max(t => t.Value!.Value); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "terminator stopped after {0} of 120 trials; best CV accuracy {1:P2}", stopAt, stopped.BestValue)); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "the full 120-trial run: best {0:P2} — {1:+0.00;-0.00} percentage points for {2} more trials", + full.BestValue, 100 * (full.BestValue - bestAtStop), 120 - stopAt)); +var folds = (IEnumerable)stopped.BestTrial.SystemAttrs["terminator:cv_scores"]!; +Console.WriteLine("the best trial's fold scores (the noise the terminator compares against): " + + string.Join(", ", folds.Select(v => Convert.ToDouble(v, CultureInfo.InvariantCulture).ToString("P1", CultureInfo.InvariantCulture)))); + +/// Cross-validates one scorecard configuration and reports its fold scores to the terminator. +/// The trial proposing C, the feature expansion and the iteration budget. +/// Customer features. +/// Churn labels. +/// Mean 5-fold accuracy. +static double Objective(Trial t, double[][] x, double[] y) +{ + double c = t.SuggestFloat("C", 1e-3, 1e3, log: true); + bool quadratic = t.SuggestCategorical("quadratic", new[] { false, true }); + long iterations = t.SuggestInt("iterations", 20, 300, log: true); + var rows = quadratic ? x.Select(r => r.Concat(r.Select(v => v * v)).Append(r[0] * r[2]).Append(r[1] * r[3]).ToArray()).ToArray() : x; + var scores = new List(); + for (int f = 0; f < 5; f++) + scores.Add(FoldAccuracy(rows, y, f, c, (int)iterations)); + TerminatorFunctions.ReportCrossValidationScores(t, scores); // what CrossValidationErrorEvaluator reads + return scores.Average(); +} + +/// Trains on four folds by gradient descent and scores the fifth. +/// Rows. +/// Labels. +/// The held-out fold (rows with index % 5 == fold). +/// Inverse L2 strength. +/// Gradient-descent steps. +/// Held-out accuracy. +static double FoldAccuracy(double[][] x, double[] y, int fold, double c, int iterations) +{ + int d = x[0].Length, nTrain = x.Length - x.Length / 5; + var w = new double[d]; + double b = 0; + for (int it = 0; it < iterations; it++) + { + var g = new double[d]; + double gb = 0; + for (int i = 0; i < x.Length; i++) + { + if (i % 5 == fold) continue; + double err = 1 / (1 + Math.Exp(-(Dot(w, x[i]) + b))) - y[i]; + for (int j = 0; j < d; j++) g[j] += err * x[i][j]; + gb += err; + } + for (int j = 0; j < d; j++) w[j] -= 0.5 * (g[j] / nTrain + w[j] / (c * nTrain)); + b -= 0.5 * gb / nTrain; + } + int correct = 0, total = 0; + for (int i = fold; i < x.Length; i += 5) + { + correct += ((Dot(w, x[i]) + b) > 0 ? 1.0 : 0.0) == y[i] ? 1 : 0; + total++; + } + return (double)correct / total; + + static double Dot(double[] w, double[] r) + { + double s = 0; + for (int j = 0; j < w.Length; j++) s += w[j] * r[j]; + return s; + } +} + +/// The synthetic telecom customer table. +static class Customers +{ + /// Generates standardized features and churn labels; the same seed gives the same table. + /// Customer count. + /// Data seed. + /// Ten features per customer and a 0/1 churn label. + public static (double[][] X, double[] Y) Generate(int rows, int seed) + { + var rng = new Random(seed); // data generation only — samplers never use System.Random + var x = new double[rows][]; + var y = new double[rows]; + for (int i = 0; i < rows; i++) + { + var r = Enumerable.Range(0, 10).Select(_ => Math.Sqrt(-2 * Math.Log(1 - rng.NextDouble())) * Math.Cos(2 * Math.PI * rng.NextDouble())).ToArray(); + double z = -0.8 + 1.1 * r[0] - 0.9 * r[1] + 0.6 * r[0] * r[2] + 0.5 * r[3] * r[3] - 0.4 * r[4]; + y[i] = rng.NextDouble() < 1 / (1 + Math.Exp(-z)) ? 1 : 0; + x[i] = r; + } + return (x, y); + } +} diff --git a/examples/README.md b/examples/README.md index df02f9f..af7f517 100644 --- a/examples/README.md +++ b/examples/README.md @@ -59,6 +59,7 @@ classes). | [`0_PressureVesselParetoReport.cs`](0_PressureVesselParetoReport.cs) | NSGA-II, mixed directions, constraints, Pareto + hypervolume charts | pressure-vessel cost vs. capacity under the ASME thickness rules (RE23) | | [`0_SpamFilterTimeBudget.cs`](0_SpamFilterTimeBudget.cs) | TPE with only a `timeout`, `show_progress_bar` | a naive-Bayes spam filter tuned inside a fixed ten-second slot | | [`0_DistributedJournalHyperband.cs`](0_DistributedJournalHyperband.cs) | 3 processes × `n_jobs=2` on one `JournalFileBackend`, `HyperbandPruner` | SGD ridge house-price model; Hyperband's brackets span all six workers through the shared file | +| [`0_ChurnScorecardTerminator.cs`](0_ChurnScorecardTerminator.cs) | `report_cross_validation_scores` + default `TerminatorCallback` | stop tuning a churn scorecard when the GP regret bound drops below the CV noise; compared with a full run | | [`0_ContainerLoadingStopAtBound.cs`](0_ContainerLoadingStopAtBound.cs) | `enqueue_trial(skip_if_exists=True)`, `study.stop()` from a callback | a bin-packing heuristic for trailer loading, warm-started from planners' rules, stopped at the lower bound | Notes from running them: From 0d97a6aafcf16221b36bf1a3809c0ca353da0632 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:17:06 +0200 Subject: [PATCH 005/145] feat(examples): 0_PricingRemoteWorkersGrpc - remote workers (C# and Python) through the gRPC storage proxy The database host runs GrpcProxyServer.MakeServer over an RDBStorage SQLite file on a free loopback port (the hosted, stoppable form of run_grpc_proxy_server); the study is created through a GrpcStorageProxy like any remote client; four .NET workers (own proxy connection each, TPE seeds 1-4) and - when python with optuna is on PATH - a Python Optuna worker using optuna.storages.GrpcStorageProxy optimize it together. Objective: a quarter's gross profit under constant- elasticity demand with a decaying promo lift and a post-promo dip. Run here: 70 trials (4 x 15 .NET + 10 Python) in one study, best $222.5k found by a .NET worker; the C# server and the Python client interoperate over the wire. No defect found. Co-Authored-By: Claude Opus 5.5 (1M context) --- examples/0_PricingRemoteWorkersGrpc.cs | 137 +++++++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 138 insertions(+) create mode 100644 examples/0_PricingRemoteWorkersGrpc.cs diff --git a/examples/0_PricingRemoteWorkersGrpc.cs b/examples/0_PricingRemoteWorkersGrpc.cs new file mode 100644 index 0000000..f17f2e8 --- /dev/null +++ b/examples/0_PricingRemoteWorkersGrpc.cs @@ -0,0 +1,137 @@ +#:project ../src/OptunaSharp.Core/OptunaSharp.Core.csproj +#:project ../src/OptunaSharp.Storage.Sqlite/OptunaSharp.Storage.Sqlite.csproj +#:project ../src/OptunaSharp.Storage.Grpc/OptunaSharp.Storage.Grpc.csproj +#:property PublishAot=false + +// Remote workers through Optuna's gRPC storage proxy — the database stays on one host behind +// `run_grpc_proxy_server`; four .NET workers that only know the proxy's address (GrpcStorageProxy) tune a +// retailer's promotion pricing in parallel, and — when a Python with optuna is on PATH — a Python Optuna +// worker joins the same study over the same wire protocol. +// +// source: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.storages.run_grpc_proxy_server.html +// https://optuna.readthedocs.io/en/stable/reference/generated/optuna.storages.GrpcStorageProxy.html +// (the proxy pair: one process owns the RDB storage, workers talk gRPC) +// +// Analytic objective, a standard retail model: weekly demand with constant price elasticity (−2.1), a +// promotion lift that grows with discount depth but decays over a long promo (customers stock up), a +// post-promo dip, and a margin floor. The knobs: list price, discount depth, promo length and how often to +// run it per quarter; the score is the quarter's gross profit in $k (maximized). The proxy fronts a SQLite +// file, so the finished study can also be opened directly afterwards. +// +// Run: dotnet run examples/0_PricingRemoteWorkersGrpc.cs + +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using OptunaSharp; +using OptunaSharp.Logging; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; +using OptunaSharp.Storages.Grpc; + +OptunaLogging.SetVerbosity(LogLevel.Warning); // silence the per-trial INFO lines + +// 1. the database host: SQLite behind the gRPC proxy, on a free local port. +string db = Path.Combine(Path.GetTempPath(), "optunasharp-pricing.db"); +File.Delete(db); +using var sqlite = new RDBStorage("sqlite:///" + db.Replace('\\', '/')); +int port = FreePort(); +await using var server = GrpcProxyServer.MakeServer(sqlite, host: "127.0.0.1", port: port); +await server.StartAsync(); +Console.WriteLine($"gRPC storage proxy listening on 127.0.0.1:{port}, backed by {db}"); + +// 2. the study is created through the proxy, like any remote client would. +using (var admin = new GrpcStorageProxy("127.0.0.1", port)) + Optuna.CreateStudy(storage: admin, studyName: "q3-promo-pricing", direction: StudyDirection.Maximize); + +// 3. four .NET workers, each with its own proxy connection, plus an optional Python worker. +var pythonWorker = Task.Run(() => PythonWorker(port)); +var dotnetWorkers = Enumerable.Range(1, 4).Select(id => Task.Run(() => +{ + using var proxy = new GrpcStorageProxy("127.0.0.1", port); + var study = Optuna.LoadStudy("q3-promo-pricing", proxy, sampler: new TPESampler(seed: id, nStartupTrials: 10)); + study.Optimize(t => { t.SetUserAttr("worker", $".NET #{id}"); return Promo.QuarterProfit(t); }, nTrials: 15); +})).ToArray(); +await Task.WhenAll(dotnetWorkers); +string pythonNote = await pythonWorker; + +// 4. read the result back through the proxy. +using (var reader = new GrpcStorageProxy("127.0.0.1", port)) +{ + var study = Optuna.LoadStudy("q3-promo-pricing", reader); + var byWorker = study.Trials.GroupBy(t => (string?)t.UserAttrs.GetValueOrDefault("worker") ?? "?").Select(g => $"{g.Key}: {g.Count()}"); + Console.WriteLine($"trials: {study.Trials.Count} ({string.Join(", ", byWorker)}); {pythonNote}"); + var best = study.BestTrial; + Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "best quarter profit ${0:N1}k by {1} (trial #{2}): {3}", best.Value, best.UserAttrs["worker"], best.Number, + string.Join(", ", best.Params.Select(kv => string.Format(CultureInfo.InvariantCulture, "{0}={1:G4}", kv.Key, kv.Value))))); +} +await server.StopAsync(); + +/// Finds a free TCP port on the loopback interface. +/// A port number that was free a moment ago. +static int FreePort() +{ + var l = new TcpListener(IPAddress.Loopback, 0); + l.Start(); + int p = ((IPEndPoint)l.LocalEndpoint).Port; + l.Stop(); + return p; +} + +/// Runs ten trials from Python Optuna through the same proxy, when Python and optuna are available. +/// The proxy's port. +/// A one-line note on what the Python worker did. +static string PythonWorker(int port) +{ + string code = "import optuna,sys,math; optuna.logging.set_verbosity(optuna.logging.ERROR)\n" + + "from optuna.storages import GrpcStorageProxy\n" + + "s=optuna.load_study(study_name='q3-promo-pricing', storage=GrpcStorageProxy(host='127.0.0.1', port=int(sys.argv[1])), sampler=optuna.samplers.RandomSampler(seed=9))\n" + + "def obj(t):\n" + + " p=t.suggest_float('list_price',8,30); d=t.suggest_float('discount',0.0,0.5); w=t.suggest_int('promo_weeks',1,6); n=t.suggest_int('promos_per_quarter',0,3)\n" + + " t.set_user_attr('worker','Python')\n" + + " weeks=13; promo=min(weeks, n*w); base=1200*(p/15)**-2.1; lift=1+2.5*d*math.exp(-0.25*(w-1)); dip=1-0.35*d*min(n,promo)\n" + + " unit=max(p*(1-d)-6.0,-6.0); full=max(p-6.0,-6.0)\n" + + " return (promo*base*(1-d)**-2.1*lift*unit+(weeks-promo)*base*dip*full)/1000\n" + + "s.optimize(obj, n_trials=10)\n"; + try + { + var psi = new ProcessStartInfo("python") { RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var a in new[] { "-c", code, port.ToString(CultureInfo.InvariantCulture) }) + psi.ArgumentList.Add(a); + using var p = Process.Start(psi)!; + p.StandardOutput.ReadToEnd(); + string err = p.StandardError.ReadToEnd(); + p.WaitForExit(); + return p.ExitCode == 0 ? "a Python Optuna worker added its trials over gRPC" : "python worker failed: " + err.Trim().Split('\n').Last(); + } + catch (System.ComponentModel.Win32Exception) + { + return "no python on PATH — the Python worker was skipped"; + } +} + +/// The retail promotion model. +static class Promo +{ + /// Gross profit of one quarter (13 weeks) under the trial's pricing plan. + /// The trial proposing list price, discount depth, promo length and promo count. + /// Profit in thousands of dollars. + public static double QuarterProfit(Trial t) + { + double price = t.SuggestFloat("list_price", 8, 30); // $ per unit + double discount = t.SuggestFloat("discount", 0.0, 0.5); // promo depth + long weeksPerPromo = t.SuggestInt("promo_weeks", 1, 6); + long promos = t.SuggestInt("promos_per_quarter", 0, 3); + const int Weeks = 13; + const double UnitCost = 6.0; + double promoWeeks = Math.Min(Weeks, promos * weeksPerPromo); + double baseDemand = 1200 * Math.Pow(price / 15, -2.1); // constant elasticity around $15 + double lift = 1 + 2.5 * discount * Math.Exp(-0.25 * (weeksPerPromo - 1)); // long promos lose their pull + double dip = 1 - 0.35 * discount * Math.Min(promos, promoWeeks); // stock-up hangover + double promoUnit = Math.Max(price * (1 - discount) - UnitCost, -UnitCost); + double fullUnit = Math.Max(price - UnitCost, -UnitCost); + return (promoWeeks * baseDemand * Math.Pow(1 - discount, -2.1) * lift * promoUnit + + (Weeks - promoWeeks) * baseDemand * dip * fullUnit) / 1000; + } +} diff --git a/examples/README.md b/examples/README.md index af7f517..1a3b3ff 100644 --- a/examples/README.md +++ b/examples/README.md @@ -59,6 +59,7 @@ classes). | [`0_PressureVesselParetoReport.cs`](0_PressureVesselParetoReport.cs) | NSGA-II, mixed directions, constraints, Pareto + hypervolume charts | pressure-vessel cost vs. capacity under the ASME thickness rules (RE23) | | [`0_SpamFilterTimeBudget.cs`](0_SpamFilterTimeBudget.cs) | TPE with only a `timeout`, `show_progress_bar` | a naive-Bayes spam filter tuned inside a fixed ten-second slot | | [`0_DistributedJournalHyperband.cs`](0_DistributedJournalHyperband.cs) | 3 processes × `n_jobs=2` on one `JournalFileBackend`, `HyperbandPruner` | SGD ridge house-price model; Hyperband's brackets span all six workers through the shared file | +| [`0_PricingRemoteWorkersGrpc.cs`](0_PricingRemoteWorkersGrpc.cs) | gRPC storage proxy (`MakeServer` over SQLite) + `GrpcStorageProxy` workers | promotion pricing tuned by four .NET workers and, if available, a Python Optuna worker over the same wire | | [`0_ChurnScorecardTerminator.cs`](0_ChurnScorecardTerminator.cs) | `report_cross_validation_scores` + default `TerminatorCallback` | stop tuning a churn scorecard when the GP regret bound drops below the CV noise; compared with a full run | | [`0_ContainerLoadingStopAtBound.cs`](0_ContainerLoadingStopAtBound.cs) | `enqueue_trial(skip_if_exists=True)`, `study.stop()` from a callback | a bin-packing heuristic for trailer loading, warm-started from planners' rules, stopped at the lower bound | From b448b95b304a35cdfd760a1b38a71bdefa025a3a Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:19:19 +0200 Subject: [PATCH 006/145] feat(examples): 0_BuildingEnergyForestSearchCV - OptunaSearchCV over the managed random forest The scikit-learn-style integration as a real job: OptunaSearchCV cross-validates Core's byte-exact RandomForestRegressor port (SklearnEstimator, KFold(5, shuffle)) over n_estimators / max_depth / min_samples_leaf / max_features to predict a building's heating load from geometry and glazing (a deterministic 768-design table shaped like the classic energy-efficiency study); the refitted best forest is scored on 200 held-out designs: CV R^2 0.9972, held-out R^2 0.9972, MAE 0.33. The per-trial terminator FutureWarning is Optuna's own behavior. No defect found. Co-Authored-By: Claude Opus 5.5 (1M context) --- examples/0_BuildingEnergyForestSearchCV.cs | 104 +++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 105 insertions(+) create mode 100644 examples/0_BuildingEnergyForestSearchCV.cs diff --git a/examples/0_BuildingEnergyForestSearchCV.cs b/examples/0_BuildingEnergyForestSearchCV.cs new file mode 100644 index 0000000..22bd177 --- /dev/null +++ b/examples/0_BuildingEnergyForestSearchCV.cs @@ -0,0 +1,104 @@ +#:project ../src/OptunaSharp.Core/OptunaSharp.Core.csproj +#:property PublishAot=false + +// Building energy model with OptunaSearchCV — the scikit-learn-style integration: OptunaSearchCV +// cross-validates a random-forest regressor (OptunaSharp's managed, byte-exact port of scikit-learn's +// RandomForestRegressor) that predicts a building's heating load from its geometry and glazing, the way an +// energy consultant screens designs before running a full thermal simulation. +// +// source: https://optuna-integration.readthedocs.io/en/stable/reference/generated/optuna_integration.OptunaSearchCV.html +// (OptunaSearchCV: an estimator, `param_distributions`, `n_trials`, CV scoring, refit) +// +// Real training: a deterministic table of 768 building designs shaped like the classic energy-efficiency +// study (relative compactness, surface/wall/roof area, height, orientation, glazing area and its +// distribution) with a heating load that depends non-linearly on them; 5-fold KFold CV with R² as the score. +// The forest's depth, leaf size, feature subsampling and size are searched; the refitted best forest is then +// scored on 200 held-out designs it never saw. +// +// Run: dotnet run examples/0_BuildingEnergyForestSearchCV.cs + +using System.Globalization; +using OptunaSharp; +using OptunaSharp.Distributions; +using OptunaSharp.Integration; +using OptunaSharp.Integration.Sklearn; +using OptunaSharp.Logging; + +OptunaLogging.SetVerbosity(LogLevel.Warning); // silence the per-trial INFO lines + +var (x, y) = Buildings.Generate(rows: 968, seed: 47); +var (xTrain, yTrain) = (Slice(x, 0, 768), y[..768]); +var (xTest, yTest) = (Slice(x, 768, 968), y[768..]); + +var search = new OptunaSearchCV( + new SklearnEstimator(new RandomForestRegressor(randomState: 0), xTrain, yTrain, cv: new KFold(5, shuffle: true, randomState: 0)), + new Dictionary + { + ["n_estimators"] = new IntDistribution(20, 120, step: 20), + ["max_depth"] = new IntDistribution(2, 20), + ["min_samples_leaf"] = new IntDistribution(1, 20, log: true), + ["max_features"] = new FloatDistribution(0.2, 1.0), + }, + nTrials: 25, + randomState: 0); +search.Fit(); + +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "best 5-fold CV R²: {0:F4} after {1} trials", search.BestScore, search.Study.Trials.Count)); +foreach (var kv in search.BestParams) + Console.WriteLine($" {kv.Key,-17} {kv.Value}"); +var forest = new RandomForestRegressor( + nEstimators: (int)(long)search.BestParams["n_estimators"]!, maxDepth: (int)(long)search.BestParams["max_depth"]!, + minSamplesLeaf: (int)(long)search.BestParams["min_samples_leaf"]!, maxFeatures: search.BestParams["max_features"], randomState: 0); +forest.Fit(xTrain, yTrain); +double[] pred = forest.Predict(xTest); +double mean = yTest.Average(); +double r2 = 1 - yTest.Zip(pred, (a, p) => (a - p) * (a - p)).Sum() / yTest.Sum(a => (a - mean) * (a - mean)); +double mae = yTest.Zip(pred, (a, p) => Math.Abs(a - p)).Average(); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "held-out designs: R² {0:F4}, mean absolute error {1:F2} kWh/m²", r2, mae)); + +/// Copies a block of rows out of a matrix. +/// The matrix. +/// First row (inclusive). +/// Last row (exclusive). +/// The rows as a new matrix. +static double[,] Slice(double[,] m, int from, int to) +{ + var o = new double[to - from, m.GetLength(1)]; + for (int i = from; i < to; i++) + for (int j = 0; j < m.GetLength(1); j++) + o[i - from, j] = m[i, j]; + return o; +} + +/// The synthetic building-design table. +static class Buildings +{ + /// Generates designs and heating loads; the same seed gives the same table. + /// Number of designs. + /// Data seed. + /// Eight features per design and its heating load in kWh/m². + public static (double[,] X, double[] Y) Generate(int rows, int seed) + { + var rng = new Random(seed); // data generation only — samplers never use System.Random + var x = new double[rows, 8]; + var y = new double[rows]; + for (int i = 0; i < rows; i++) + { + double compactness = 0.62 + 0.36 * rng.NextDouble(); + double surface = 515 + 300 * (1 - compactness) / 0.36 + 10 * rng.NextDouble(); + double wall = 245 + 170 * rng.NextDouble(); + double roof = Math.Max(110, (surface - wall) / 2); + double height = compactness > 0.75 ? 7.0 : 3.5; + double orientation = 2 + rng.Next(4); + double glazing = new[] { 0.0, 0.1, 0.25, 0.4 }[rng.Next(4)]; + double distribution = glazing == 0 ? 0 : 1 + rng.Next(5); + // heating load: tall compact buildings lose less per m²; glazing and roof area add losses. + y[i] = 6 + 28 * (height / 7.0) * Math.Pow(compactness, 1.5) + 0.018 * roof * (height < 5 ? 0.4 : 1.0) + + 22 * glazing + 0.6 * (distribution is 2 or 3 ? glazing * 4 : 0) + 0.2 * (orientation - 3.5) + 0.8 * (rng.NextDouble() - 0.5); + double[] r = { compactness, surface, wall, roof, height, orientation, glazing, distribution }; + for (int j = 0; j < 8; j++) + x[i, j] = r[j]; + } + return (x, y); + } +} diff --git a/examples/README.md b/examples/README.md index 1a3b3ff..00bb967 100644 --- a/examples/README.md +++ b/examples/README.md @@ -59,6 +59,7 @@ classes). | [`0_PressureVesselParetoReport.cs`](0_PressureVesselParetoReport.cs) | NSGA-II, mixed directions, constraints, Pareto + hypervolume charts | pressure-vessel cost vs. capacity under the ASME thickness rules (RE23) | | [`0_SpamFilterTimeBudget.cs`](0_SpamFilterTimeBudget.cs) | TPE with only a `timeout`, `show_progress_bar` | a naive-Bayes spam filter tuned inside a fixed ten-second slot | | [`0_DistributedJournalHyperband.cs`](0_DistributedJournalHyperband.cs) | 3 processes × `n_jobs=2` on one `JournalFileBackend`, `HyperbandPruner` | SGD ridge house-price model; Hyperband's brackets span all six workers through the shared file | +| [`0_BuildingEnergyForestSearchCV.cs`](0_BuildingEnergyForestSearchCV.cs) | `OptunaSearchCV` + Core's managed `RandomForestRegressor`, `KFold` | heating-load model for screening building designs; refitted forest scored on held-out designs | | [`0_PricingRemoteWorkersGrpc.cs`](0_PricingRemoteWorkersGrpc.cs) | gRPC storage proxy (`MakeServer` over SQLite) + `GrpcStorageProxy` workers | promotion pricing tuned by four .NET workers and, if available, a Python Optuna worker over the same wire | | [`0_ChurnScorecardTerminator.cs`](0_ChurnScorecardTerminator.cs) | `report_cross_validation_scores` + default `TerminatorCallback` | stop tuning a churn scorecard when the GP regret bound drops below the CV noise; compared with a full run | | [`0_ContainerLoadingStopAtBound.cs`](0_ContainerLoadingStopAtBound.cs) | `enqueue_trial(skip_if_exists=True)`, `study.stop()` from a callback | a bin-packing heuristic for trailer loading, warm-started from planners' rules, stopped at the lower bound | From 9680873b5e1dba79eefd926ec479599b8ac724ff Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 20:22:03 +0200 Subject: [PATCH 007/145] feat(examples): 0_ConcreteMixDesignGp - GPSampler on a mixed stepped/int/categorical space with constraints Concrete mix design as Bayesian optimization: cement content (5 kg steps), water-cement ratio (0.01 steps), slag and fly-ash replacement (int, step 5), superplasticizer dose (continuous) and the aggregate type (categorical), maximizing 28-day strength from Abrams' law with SCM and aggregate factors, under SetConstraint cost <= $95/m^3 and slump >= 75 mm. 12 random pours then 28 GP-planned: 57.1 MPa (both constraints nearly active) vs 44.7 MPa best random. Live parity probe (40 trials): every stepped, integer and categorical choice is identical to Optuna 5.0.0's GPSampler; the continuous dose differs by <= 2.1e-4 relative from trial 24 on - the documented GP / L-BFGS-B tolerance class. No defect found. Co-Authored-By: Claude Opus 5.5 (1M context) --- examples/0_ConcreteMixDesignGp.cs | 69 +++++++++++++++++++++++++++++++ examples/README.md | 1 + 2 files changed, 70 insertions(+) create mode 100644 examples/0_ConcreteMixDesignGp.cs diff --git a/examples/0_ConcreteMixDesignGp.cs b/examples/0_ConcreteMixDesignGp.cs new file mode 100644 index 0000000..60f2d87 --- /dev/null +++ b/examples/0_ConcreteMixDesignGp.cs @@ -0,0 +1,69 @@ +#:project ../src/OptunaSharp.Core/OptunaSharp.Core.csproj +#:property PublishAot=false + +// Concrete mix design with Bayesian optimization over a mixed space — GPSampler picks the cement content +// (5 kg steps), water–cement ratio (0.01 steps), slag and fly-ash replacement, superplasticizer dose and the +// aggregate type to maximize 28-day compressive strength, under a cost cap and a minimum-workability rule: +// each trial batch is a lab pour, so few trials matter. +// +// source: https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.GPSampler.html +// (GP sampling over float — including stepped — int and categorical parameters, with constraints) +// https://en.wikipedia.org/wiki/Water%E2%80%93cement_ratio (Abrams' law: strength falls with w/c) +// +// Surrogate of the lab, built on the textbook relations: Abrams' law f = A / B^(w/c) for the paste, a +// cement-content factor, slag that costs early strength but adds to it by 28 days while fly ash mostly +// dilutes it, superplasticizer that buys workability (so a lower w/c is placeable), and an aggregate factor +// (crushed granite > river gravel > recycled). Constraints, via `SetConstraint`: material cost ≤ $95/m³ and +// slump ≥ 75 mm. 12 random pours, then 28 GP-planned ones. +// +// Run: dotnet run examples/0_ConcreteMixDesignGp.cs + +using System.Globalization; +using OptunaSharp; +using OptunaSharp.Logging; +using OptunaSharp.Samplers; + +OptunaLogging.SetVerbosity(LogLevel.Warning); // silence the per-trial INFO lines + +var study = Optuna.CreateStudy(sampler: new GPSampler(seed: 50, nStartupTrials: 12), direction: StudyDirection.Maximize, studyName: "c40-mix"); +study.Optimize(Objective, nTrials: 40); + +var best = study.BestTrial; // best FEASIBLE mix +int feasible = study.Trials.Count(t => t.Constraints.Values.All(c => c <= 0)); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0} pours, {1} within cost and slump; best 28-day strength {2:F1} MPa (pour #{3})", + study.Trials.Count, feasible, best.Value, best.Number)); +foreach (var kv in best.Params) + Console.WriteLine(string.Format(CultureInfo.InvariantCulture, " {0,-20} {1}", kv.Key, kv.Value)); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, " cost ${0:F1}/m³, slump {1:F0} mm", best.UserAttrs["cost"], best.UserAttrs["slump_mm"])); +var randomBest = study.Trials.Take(12).Where(t => t.Constraints.Values.All(c => c <= 0)).Select(t => t.Value ?? 0).DefaultIfEmpty(0).Max(); +Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "best of the 12 random pours: {0:F1} MPa", randomBest)); + +/// "Pours" one mix: predicts its 28-day strength, cost and slump. +/// The trial proposing the mix. +/// 28-day compressive strength in MPa; cost and slump limits are recorded as constraints. +static double Objective(Trial t) +{ + double cement = t.SuggestFloat("cement_kg", 250, 500, step: 5); // binder per m³, 5 kg steps + double wc = t.SuggestFloat("water_cement", 0.30, 0.65, step: 0.01); // water–cement ratio + long slagPct = t.SuggestInt("slag_pct", 0, 50, step: 5); // GGBS replacing cement + long ashPct = t.SuggestInt("fly_ash_pct", 0, 30, step: 5); // fly ash replacing cement + double sp = t.SuggestFloat("superplasticizer_kg", 0, 8); // admixture dose + string aggregate = t.SuggestCategorical("aggregate", new[] { "crushed_granite", "river_gravel", "recycled" }); + + double clinkerShare = Math.Max(0.2, 1 - (slagPct + ashPct) / 100.0); + double abrams = 96.5 / Math.Pow(7.0, wc); // MPa, Abrams' law + double cementFactor = 0.75 + 0.25 * Math.Min(1, cement / 400); + double scm = clinkerShare + 0.85 * slagPct / 100.0 + 0.45 * ashPct / 100.0; // 28-day contribution + double aggFactor = aggregate switch { "crushed_granite" => 1.06, "river_gravel" => 1.0, _ => 0.88 }; + double strength = abrams * cementFactor * scm * aggFactor; + + double water = wc * cement; + double slump = 20 + 0.55 * (water - 140) + 22 * sp + 0.6 * ashPct - (aggregate == "crushed_granite" ? 15 : 0); + double cost = cement * clinkerShare * 0.14 + cement * slagPct / 100.0 * 0.09 + cement * ashPct / 100.0 * 0.05 + + sp * 2.8 + (aggregate switch { "crushed_granite" => 24, "river_gravel" => 20, _ => 14 }); + t.SetUserAttr("cost", cost); + t.SetUserAttr("slump_mm", slump); + t.SetConstraint("cost_cap", cost - 95.0); // ≤ $95 per m³ + t.SetConstraint("workability", 75.0 - slump); // ≥ 75 mm slump + return strength; +} diff --git a/examples/README.md b/examples/README.md index 00bb967..6b5593b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -59,6 +59,7 @@ classes). | [`0_PressureVesselParetoReport.cs`](0_PressureVesselParetoReport.cs) | NSGA-II, mixed directions, constraints, Pareto + hypervolume charts | pressure-vessel cost vs. capacity under the ASME thickness rules (RE23) | | [`0_SpamFilterTimeBudget.cs`](0_SpamFilterTimeBudget.cs) | TPE with only a `timeout`, `show_progress_bar` | a naive-Bayes spam filter tuned inside a fixed ten-second slot | | [`0_DistributedJournalHyperband.cs`](0_DistributedJournalHyperband.cs) | 3 processes × `n_jobs=2` on one `JournalFileBackend`, `HyperbandPruner` | SGD ridge house-price model; Hyperband's brackets span all six workers through the shared file | +| [`0_ConcreteMixDesignGp.cs`](0_ConcreteMixDesignGp.cs) | `GPSampler` over stepped floats, ints and a categorical + 2 constraints — discrete choices identical to Optuna | concrete mix design: 28-day strength under a cost cap and a workability floor | | [`0_BuildingEnergyForestSearchCV.cs`](0_BuildingEnergyForestSearchCV.cs) | `OptunaSearchCV` + Core's managed `RandomForestRegressor`, `KFold` | heating-load model for screening building designs; refitted forest scored on held-out designs | | [`0_PricingRemoteWorkersGrpc.cs`](0_PricingRemoteWorkersGrpc.cs) | gRPC storage proxy (`MakeServer` over SQLite) + `GrpcStorageProxy` workers | promotion pricing tuned by four .NET workers and, if available, a Python Optuna worker over the same wire | | [`0_ChurnScorecardTerminator.cs`](0_ChurnScorecardTerminator.cs) | `report_cross_validation_scores` + default `TerminatorCallback` | stop tuning a churn scorecard when the GP regret bound drops below the CV noise; compared with a full run | From 5e5ba530d01d8d398e4729d80c1ac044b0f36a13 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 21:50:21 +0200 Subject: [PATCH 008/145] docs(plans): plan 13, Rustuna-derived performance: compatible speedups as the only path, opt-in switches where results would change The owner asked for a plan to integrate Rustuna's optimizations, with an evaluation of where a dual implementation is warranted (a user choosing Optuna's path or Rustuna's). Rustuna 0.2.1, the Optuna team's Rust port, runs its announcement's workloads 3-60x faster than OptunaSharp (TPE 7x, CMA-ES 100k 60x, NSGA-II 3-4x), while OptunaSharp is 3-5x faster than Optuna 5.0 and byte-identical to it. Decisions recorded (owner, 2026-11-08, through two rounds of design questions): - Rule: "prefer rustuna unless it affects results". A Rustuna technique that keeps every observable result becomes the only implementation; one that changes results is measured and decided per item. - CMA-ES reusing the optimizer it just told (Rustuna's live object): opt-in, Optuna's path the default. - TPE fast numerics: decided after the byte-identical fused kernel exists and can be measured. - Separate trial queue and trial discarding: deferred. The owner asked whether Rustuna has a TrialQueue (it does: an in-memory default plus SQLite3 and directory backends, Redis planned). For discarding the owner asked for "what's best + don't break Optuna compatibility", so the plan proposes compact storage of finished trials first. Evidence in the plan, all measured in session 5417a82e on the owner's i9-13900K (Release, throttling opt-out): - A scratch prototype of the CMA-ES part (a patched copy of d0558f5) took 10k trials from 18 s to 2.6 s and 100k from 812 s to 26 s (single-threaded OpenBLAS), byte-identical: five seeded studies hashed over every trial and its stored pickle, 86 unit and 29 live-Python interop CMA tests, mixed C#/Python fleet included. Its three changes: an optimizer cache dropped after each tell, a folded history, zero-copy reads. - How different the result-changing paths are. Keeping the told CMA-ES optimizer diverges from generation 3 (eigenvector sign flips make the first difference large), with the same quality over 10 seeds and about 15 % more speed. TPE scoring noise up to 1e-6 changed no decision in three 1,000-trial runs (smallest gap 3.4e-4); MOTPE stayed identical up to 1e-9. The queue is identical in one process; discarding changes study.Trials. - Profiles behind each item: copy-on-write storage clones 66 % of NSGA-II and 23 % of MOTPE; the MOTPE split, recomputed per parameter, 24 %; TPE scoring 58 %. The plan: an inventory of 12 same-result items (S1-S12), 4 two-path items (D1-D4) and 6 rejected ones, each with Rustuna's code reference, our current code and the evidence. Then the rule for one path or two, the switch design (on the component, named for the behavior), phases 0-8, gates (Python as the oracle on every CI OS, concurrency suites, Release benchmarks, both paths tested), targets and risks. docs/plans/README.md lists plan 13 and describes the performance workstream. --- docs/plans/13-rustuna-derived-performance.md | 348 +++++++++++++++++++ docs/plans/README.md | 9 + 2 files changed, 357 insertions(+) create mode 100644 docs/plans/13-rustuna-derived-performance.md diff --git a/docs/plans/13-rustuna-derived-performance.md b/docs/plans/13-rustuna-derived-performance.md new file mode 100644 index 0000000..9b46b42 --- /dev/null +++ b/docs/plans/13-rustuna-derived-performance.md @@ -0,0 +1,348 @@ +# Plan 13 — Rustuna-derived performance: compatible speedups first, opt-in paths where results would change + +> **Status: DESIGN (2026-11-08).** Nothing here is built yet. A scratch prototype of the CMA-ES part exists (session +> `5417a82e`, a patched copy of `d0558f5`) and is measured below; Phase 1 ports it. Background, benchmarks and the +> full Rustuna read-through: catalog note `optunasharp/notes/rustuna-comparison-2026-11-08.md`. + +--- + +## 1. The request and the decisions (owner, 2026-11-08) + +The request: "Write a plan for rustuna's optimization's integration, with evaluation for dual-impl recommendation +(user chooses if we use Optuna's path or Rustuna)." + +[Rustuna](https://github.com/optuna/rustuna) is the Optuna team's Rust port (0.2.1 on PyPI). On its announcement's +workloads it is 3-60x faster than OptunaSharp, which is itself 3-5x faster than Optuna 5.0 (§2.1). Most of the gap is +implementation, not anything Optuna compatibility requires: a scratch prototype took CMA-ES at 100,000 trials from +812 s to 26 s with byte-identical results (§2.3). + +| Question | Answer | +|---|---| +| Default when a Rustuna-style path would change results or storage contents | "We would prefer rustuna unless it affects results." So a Rustuna-style change that keeps every result becomes the only implementation; one that changes results needs a measurement of how different (§2.4) and a decision per item | +| CMA-ES keeping the optimizer it just told (Rustuna's live object) | Opt-in; Optuna's path stays the default (§5.1) | +| TPE fast numerics in candidate scoring | Decide after the exact fused kernel exists and can be measured (§5.2) | +| Separate trial queue (Rustuna's `TrialQueue`) | Deferred. The owner asked whether Rustuna has one: it does (§5.3) | +| Trial discarding (Rustuna's `apply_discard`) | Deferred: "what's best + don't break Optuna compatibility". §5.4 proposes compacting finished trials in memory instead, which keeps every API result | +| Where a user flips a switch | Deferred. The CMA-ES choice above was made on a page that showed a constructor parameter; §5.5 proposes that shape for every switch | + +## 2. Evidence + +### 2.1 Where we stand + +i9-13900K, Windows 11, Release, every process opted out of power throttling; OptunaSharp `d0558f5`, Rustuna 0.2.1, +Optuna 5.0.0; 40 float parameters in [-10, 10], seed 0, default samplers. + +| Workload | OptunaSharp | Rustuna 0.2.1 | Optuna 5.0 | +|---|---|---|---| +| TPE, 1,000 trials | 9.94 s | 1.36 s | 35.2 s | +| MOTPE (2 objectives), 1,000 trials | 36.6 s | 6.30 s | 163 s | +| TPE, 1,000 trials, 4 threads | 9.29 s shared / 2.77 s PerWorker | 0.40 s | 32.5 s | +| CMA-ES, 10,000 trials | 18.3 s | 1.59 s | 56.9 s | +| CMA-ES, 100,000 trials | 812 s | 13.6 s | not run | +| NSGA-II, 10,000 trials | 3.51 s | 0.82 s | 44.2 s | +| NSGA-II, 100,000 trials | 62.9 s | 19.4 s | not run | + +Rustuna is API-compatible, not result-compatible: its seeded trials differ from Optuna's from the first trial (its +own RNG, `rand::StdRng`), while OptunaSharp's are identical to Optuna's. + +### 2.2 Where the time goes (dotnet-trace, `dotnet-sampled-thread-time`) + +| Workload | Profile | +|---|---| +| CMA-ES, 10k | 88 % per ask: eigendecomposition 40 % (the optimizer is rebuilt from its pickle every ask and loses its cached decomposition), restore 21 %, generation scan 17 % (O(n) per ask) | +| NSGA-II, 10k | ~66 % in `Trial.SuggestFloat`: `InMemoryStorage.SetTrialParam` clones the trial's dictionaries on every suggest (Optuna's copy-on-write), O(d²) per trial; GC pauses 22 % | +| TPE, 1k | `MixtureOfProductDistribution.LogPdf` 58 %, with 4.4 GB of candidate × kernel × dimension temporaries per 1,000 trials | +| MOTPE, 300 trials (3.87 s) | the trial split 0.93 s (24 %: `FastNonDominationRank` 0.53 s, `UniqueRows` 0.39 s), recomputed for each of the 40 parameters; `SetTrialParam` 0.89 s (23 %); scoring 0.95 s | + +### 2.3 The CMA-ES prototype (scratch copy of `d0558f5`) + +`Samplers/CmaEsSampler.cs` changed in three ways, and one line of `SearchSpace/IntersectionSearchSpace.cs`: + +1. **Optimizer cache.** The optimizer deserialized from the latest stored state is kept, keyed by the trial that holds + that state (a finished trial cannot change). Later asks of the generation reuse it, eigendecomposition included. + After every tell it is dropped, so the next ask deserializes the state just stored, as Optuna does. +2. **Folded history.** Instead of copying and filtering every trial on every ask, the sampler folds the leading run of + finished trials once, in trial order, into a completed list, a by-generation index and the latest state holder. Each + ask classifies only the trials after the first unfinished one. +3. **Zero-copy reads.** The sampler and `IntersectionSearchSpace.Calculate` read `Study.GetTrialsView()`, the window the + genetic samplers already use, instead of a full copy of the history. + +| CMA-ES, 40-D, seed 0 | `d0558f5` | prototype | prototype, `OPENBLAS_NUM_THREADS=1` | Rustuna 0.2.1 | +|---|---|---|---|---| +| 10,000 trials | 17.7-18.9 s | 3.3-5.2 s | 2.56-2.59 s | 1.4-1.65 s | +| 100,000 trials | 812 s, 3.4 GB peak | 33.0-34.2 s, 31 GB allocated, 15 gen-2 GCs | 26.0 s | 13.4-13.9 s | + +**Byte-identical**, three ways: +- Five seeded studies were hashed over every trial's number, state, values, params and system attributes (the stored + `cma:optimizer:N` pickle chunks included): 40-D CMA 3,000 trials, SepCMA, with-margin, lr-adapt, and 8-D where + every 7th trial fails. The hashes equal the unmodified build's. +- 86/86 CMA-ES, pickle and search-space unit tests pass, as on the unmodified build. +- 29/29 live-Python interop tests pass, including C# and Python workers taking turns on one study, with the engine + required. + +**The trap.** Reusing the *told* object instead of reloading the stored state moved seeded results. Optuna's pickle +stores only the upper triangle of the covariance matrix and mirrors it on load; the told matrix keeps the +floating-point asymmetry of the update. SepCMA, which stores its diagonal exactly, did not move. + +**OpenBLAS.** At `nJobs: 1`, `TrialExecutorSnapshot.BlasThreadsFor` leaves OpenBLAS at its default thread count, and +the trace shows about three cores of unmanaged time beside the one optimizing thread. With `OPENBLAS_NUM_THREADS=1` +the unmodified 3,000-trial study ran in 5.6 s instead of 8.3-9.0 s, with the same hashes. + +### 2.4 How different the result-changing paths are (measured 2026-11-08) + +**CMA-ES keeping the told optimizer**, against the compatible cache, 10 seeds × 3,000 trials: +- 40-D sphere: the first trial that differs is trial 32 (generation 3) in every seed. The first difference can already + be large (a parameter can change sign): a last-bit change in the covariance can flip an eigenvector's sign, so the + same random draw maps to a different point of the same distribution. Best value, told / compatible: median 0.79, + range 0.51-1.48. Same quality, a different run. +- 10-D Rastrigin: the first difference is at trial 22, 4e-14 to 7e-12 relative, and the best value is identical in all + 10 seeds. +- Storage is unchanged: the pickle is still written every generation, so Python workers and resuming still work; + they just don't reproduce the C# run exactly. +- Speed with single-threaded OpenBLAS: 10k 2.57 → 2.18 s, 100k 26.0 → 21.9 s (about 15 %). With the default thread + count the gap is larger (10k 4.7-5.2 → 2.6 s, 100k 33.0 → 25.3 s), because each skipped eigendecomposition is a + multi-threaded one. + +**TPE fast numerics**, simulated: noise of a chosen size was added to every acquisition value, drawn from its own +counter so the sampler's random stream was untouched. +- TPE, 40-D multivariate, 1,000 trials, 3 seeds (990 decisions each): noise of 1e-12, 1e-9 and even 1e-6 changed no + decision. The smallest gap between the best and second-best candidate was 3.4e-4 (median 1.2; acquisition values + up to 89). +- MOTPE, 40 parameters, 300 trials, 2 seeds (11,600 decisions each): identical with 1e-12 and 1e-9; 1e-6 changed + decisions (first difference at trial 99 and trial 244). Smallest gaps 1.6e-7 and 3.8e-7. +- A reordered sum here errs by about 1e-13, so these runs had a margin of 5-9 orders of magnitude. That is not a + guarantee: one near-tie in a long run flips one decision, and from there the run differs from Python. Noise in + candidate *sampling* (not tested) would change stored parameters in the last bit. + +**Trial queue** (no numbers): in one process it is identical, since Rustuna also creates the WAITING trial at enqueue +time and first-in-first-out equals Optuna's lowest-number-first pop. Across processes, an in-memory queue is private; +a SQLite queue is shared and pops atomically. + +**Trial discarding** (no numbers): sampling is unchanged when only trials no sampler needs are dropped (CMA-ES reads +only the latest state holder and the current generation). `study.Trials`, the trials dataframe and the best trial (if +the best was dropped) change. + +## 3. The rule: one implementation or two + +The owner's rule, applied to each change: + +1. **Does the Rustuna-style path change anything observable?** Observable means any of: + - seeded parameters or values, compared with Python Optuna; + - bytes written to storage; + - what public APIs return (`Study.Trials`, the best trial, attributes, counts); + - behavior across processes or with Python workers sharing the storage; + - which inputs raise errors, and when. +2. **No** → one implementation. The faster path replaces the Optuna-structured code; there is nothing for a user to + choose between. The gate is §6's "no observable change". +3. **Yes** → measure how different (trajectory divergence, quality over seeds, cross-process semantics), then the owner + decides per item. A path that changes results defaults to Optuna's behavior so far (§1); the other path is a switch + on the component it changes (§5.5). +4. **Build a second path only if it pays.** It needs a measured gain on a §7 workload and no compatible change that + gets most of it. Otherwise it is not built: two paths mean two code paths to test and document forever. + +## 4. Inventory and verdicts + +"Same results" items are a single implementation (rule 2). Rustuna references are to `K:/source/rustuna` at `8acb2dd`. + +| ID | Change | Rustuna does | OptunaSharp today | Evidence | Verdict | +|---|---|---|---|---|---| +| S1 | CMA-ES optimizer cache, dropped after each tell | keeps the live `cmaes.CMA` object (`rustuna_pyo3/src/sampler/cmaes.rs`) | restores from the pickle every ask | §2.3 (measured) | same results: build (Phase 1) | +| S2 | Folded history for CMA-ES | keeps the generation's trial ids; `StudyCache` folds with a cursor | full copy + scan every ask | §2.3 (measured) | same results: build (Phase 1) | +| S3 | Zero-copy history reads in the search spaces and the remaining samplers | samplers read `&PersistedTrial` under the storage lock | `IntersectionSearchSpace`, `GroupDecomposedSearchSpace`, QMC and CMA-ES copy the list per call | §2.3 for the intersection space | same results: build (Phases 1, 3) | +| S4 | In-place writes to a running trial, snapshots built only when read | `set_trial_param` inserts in place; `get_trials` clones for callers | `InMemoryStorage.SetTrialParam` clones the trial's dictionaries per suggest | §2.2: 66 % NSGA-II, 23 % MOTPE, 12 % CMA-ES | same results: build (Phase 2) | +| S5 | MOTPE split cache | `split_cache` per (completed trial numbers, gamma) | split recomputed per parameter; only the HSSP is cached (as in Optuna) | §2.2: 24 % of MOTPE | same results: build (Phase 3) | +| S6 | TPE exact fused kernel | accumulates each candidate into one n-kernel vector with precomputed kernel constants | builds candidate × kernel × dimension matrices | §2.2: 58 % of TPE, 4.4 GB per 1k trials | same results if NumPy's operation order is kept: build (Phase 3) | +| S7 | Interning of parameter names and distributions | interned keys; categorical labels stored once per study | a new key string per suggest when computed; distribution objects per trial | memory: CMA-ES history 2.5 GB at 100k | same results: build (Phase 2) | +| S8 | Bulk attribute writes (`SetUserAttrs`) | `set_trial_attrs` / `set_study_attrs` in bulk | one storage call per attribute | not measured | added API: build (Phase 4) | +| S9 | Lazy attribute decoding in database storages | attributes are strings (no JSON) | JSON parsed per attribute at load | Rustuna's motive: 100,000 `json.loads` to load 10k trials × 10 attrs | same values; errors surface on access: evaluate (Phase 4) | +| S10 | Incremental reads in `CachedStorage` | `CachedStorage` wrapper returns references after a sync | `GetTrialsPrefixView` falls back to a full copy | not measured | same results: build (Phase 4) | +| S11 | OpenBLAS thread policy at `nJobs: 1` | n/a | default thread count | §2.3: 8.3-9.0 → 5.6 s, same hashes | same results; GP's large matrices need measuring first (Phase 5) | +| S12 | Compact storage of finished trials | `PersistedTrial` keeps internal values only | per-trial boxed dictionaries | 2.5 GB at 100k CMA-ES | same results: the compatible answer to discarding (Phase 6) | +| D1 | CMA-ES reuses the told optimizer | yes | no | §2.4: results differ from generation 3; ~15 % faster | **opt-in, Optuna default** (owner) (Phase 7) | +| D2 | TPE fast numerics in scoring | its own numerics | NumPy's | §2.4: no decision changed in measured runs; gain unknown | **decide after S6** (owner) | +| D3 | Separate trial queue | `TrialQueue`: in-memory, SQLite3, directory; Redis planned | WAITING trials in storage | §2.4 | **deferred** (owner) | +| D4 | Trial discarding | `apply_discard` + `discard_trials` | none | §2.4 | **deferred** (owner); S12 first | +| — | Process-local CMA-ES state (no pickle) | yes | no | after S1, the pickle is ~2 % of the 10k profile | rejected: breaks resuming and mixed fleets for ~2 % | +| — | ChaCha RNG | `rand::StdRng` | MT19937 | the RNG never shows in a profile | rejected: loses parity, gains nothing | +| — | String-only attributes | yes | JSON values | | rejected: an API break; S9 gets the gain | +| — | Seeding CMA-ES once instead of per ask | yes | per ask (Optuna) | negligible | rejected | +| — | Dropping pruners, callbacks, timeout, `n_jobs`, per-trial logging | yes | no | the log line is already skipped unless INFO is on | rejected | +| — | Parallel sampling under one shared sampler | fine-grained locks; the GIL released while sampling | `SamplerConcurrency.Shared` serializes; `PerWorker` samples in parallel | 4-thread TPE: 0.40 s vs 2.77 s PerWorker | no new work: `PerWorker` is the existing switch, and plan 12's no-duplicate guarantee rests on the shared lock | + +Already done before this plan: the genetic samplers fold their generations incrementally +(`BaseGASampler.GenerationTracker`, O(new trials) per ask), which is Rustuna's `generation_to_numbers`. + +### 4.1 Notes on the single-implementation items + +- **S1-S3 (Phase 1).** Port the prototype to HEAD (NumSharp 0.80.0 since), without its bisection switches and its + first-version leftovers (the string-keyed cache, the generation memo). Edge cases to test: + - trials that finish out of order wait in the tail and fold in trial order; + - a FAIL trial that told a generation (the old state is restored again); + - `considerPrunedTrials` (pruned trials folded as value-rewritten copies); + - warm start (`sourceTrials`); + - a new study on a storage that reuses ids (the fold resets when the history no longer starts with the folded trials); + - `SamplerConcurrency.PerWorker` (the clone gets an empty cache); + - a stuck RUNNING trial (the fold stops there; speed degrades toward today's, results stay right). +- **S4 (Phase 2).** Keep a mutable draft for each running trial inside `InMemoryStorage`. A write changes the draft in + O(1) and marks it dirty; a read (`GetTrial`, `GetAllTrials`, a `TrialsPrefixView` index) publishes an immutable + `FrozenTrial` snapshot from a dirty draft, under the storage lock, and caches it until the next write. A sequential + trial does 40 writes and a few reads, so O(d) instead of O(d²). + - The view then takes the storage lock only for a dirty running trial, which keeps plan 12's readers of running + trials (`HandedOutParams`) correct. + - The invariant is the one Optuna's copy-on-write protects: a snapshot a caller holds never changes. Tests assert it + for every path that hands one out. +- **S5 (Phase 3).** Cache the identities of the split, not snapshots: constant liar re-reads running trials, whose + parameters change between suggests. Key it by a per-study counter of finished trials that `InMemoryStorage` bumps, + so the key costs O(1); other storages fall back to keying by the list of trial numbers and states. +- **S6 (Phase 3).** Fuse `MixtureOfProductDistribution.LogPdf` / `AddContinuousLogPdf` into per (candidate, kernel) + loops without the big temporaries. Byte identity requires NumPy's order: + - the batched truncated-normal log-pdf (`_truncnorm.logpdf(...).sum(axis=-1)` in `probability_distributions.py`) + computes each term as `(_norm_logpdf(z) - log_gauss_mass) - log(scale)`, with NumPy's `z**2` on an array (a + square, not `pow`), then sums over the continuous dimensions with NumPy's pairwise summation over a contiguous + axis; replicate that per (candidate, kernel) on a stack buffer with `MathUtil.PairwiseSum`; + - precompute only the terms NumPy computes per kernel anyway (log mass, log scale); the same value computed once + instead of 24 times has the same bits; + - Rustuna's binned bandwidths (`binned_sigmas`, O(n) instead of a sort) are probably bit-identical: test that, adopt + if so. + - If some kernel cannot be made byte-identical, keep the old one for it and move the fast one into D2. +- **S9 (Phase 4).** Measure first: load time and allocations of a 10k-trial SQLite study with CMA-ES pickles. The only + observable difference is when malformed JSON raises (on access instead of on load); document it, or validate + cheaply at load. +- **S11 (Phase 5).** Measure `GPSampler` (500 and 1,000 trials) and CMA-ES (10, 40, 100 dimensions) with OpenBLAS at 1 + thread and at the default. Results are identical either way, so this is a resource policy: choose from the numbers + (one thread at `nJobs: 1`, or by matrix size). `TrialExecutorOptions.BlasThreads` already lets a user override it. +- **S12 (Phase 6).** Store finished trials compactly while every API still returns the same `FrozenTrial` contents: + - parameter values as one internal-representation `double[]` per trial against a per-study parameter index; + - distributions and names shared (S7); + - attributes as raw JSON until read (S9); + - snapshots materialized on read and cached weakly. + Measure the memory of a 100k-trial CMA-ES study before and after. This is the compatible answer to the owner's + "what's best + don't break Optuna compatibility" for discarding. + +## 5. The switches (two paths) + +### 5.1 D1: `CmaEsSampler(..., reuseToldOptimizer: false)`, decided + +The owner chose opt-in with Optuna's path as the default. +- **When true,** after a tell the told optimizer becomes the cache for the state it just stored, so the next asks skip + the restore. It is still written to storage exactly as before, so resuming and Python workers work, but no worker + reproduces the C# run. +- **API.** The parameter goes after every Optuna parameter, so positional calls ported from Python keep their meaning. + The XML doc states the consequence in its first sentence: seeded runs stop matching Python Optuna from generation 3. +- **Tests:** + - default-path parity is unchanged (the Phase 0 gates); + - with the switch on: convergence over seeds on sphere and Rastrigin (§2.4's numbers as the bar), the mixed-fleet + interop case still completes, and a failing teller still restores the old state; + - a guard test that a default-constructed sampler stays on Optuna's path. + +### 5.2 D2: TPE fast numerics, decide after S6 + +Phase 3 builds S6. Then measure two things: +- the time scoring still takes; +- a fast scoring kernel (vectorized exp and log, reordered sums), with candidate sampling left exact. + +Rerun §2.4's noise harness, extended to MOTPE with 1,000 trials and to constraints, and ask the owner with both numbers. +If built, it is `TPESampler(..., numerics: TpeNumerics.Exact)` with the default the owner picks. + +### 5.3 D3: separate trial queue, deferred + +Rustuna's design, for the record: +- A `TrialQueue` trait (`rustuna_core/src/trial_queue.rs`) with an in-memory default. +- `SQLite3TrialQueue`: a namespaced `trial_queue` table with an atomic pop. +- `DirectoryTrialQueue`: `pending/` and `processing/` files. +- A Redis backend, planned. +- `enqueue_trial` still creates the WAITING trial in storage, numbered at enqueue time, then pushes its id. `ask` pops + from the queue instead of scanning storage. +- `add_trial` with WAITING is not queued. + +Recommendation when the owner returns to it: opt-in, with Optuna's storage-backed WAITING trials as the default: +- the gain in memory is nil (our WAITING pop is already O(1) through a cursor); +- on database storages it saves one query per ask; +- a SQLite or Redis queue fixes Optuna's duplicate pops on SQLite. + +The risk of a default in-memory queue is private enqueues across processes. + +### 5.4 D4: trial discarding, deferred + +S12 comes first because it saves memory without changing any API result. Discarding is worth reopening only if S12's +measured memory is not enough. If it is reopened, the compatible slice is sampler-driven discarding of what no +sampler or API caller can need, which is hard to define for the public `Study.Trials`. Anything beyond that changes +`Study.Trials`, so it would be opt-in on `InMemoryStorage`. + +### 5.5 Where switches live (proposed; the owner deferred the general question) + +- **On the component they change:** a constructor parameter or options property of the sampler, storage or queue. + Nothing process-wide: a process default would also reach libraries in the same process (the study `OptunaSearchCV` + creates). +- **Named for the behavior, not for Rustuna:** `reuseToldOptimizer`, `TpeNumerics.Fast`, `DiscardTrials`. +- **Defaults chosen per item by the owner;** so far Optuna's behavior (§1). +- **One documentation page,** "Performance and compatibility", lists every switch: what it changes, whether seeded runs + still match Python, and whether the storage can still be shared with Python workers. + +## 6. Phases and gates + +| Phase | Content | Depends on | +|---|---|---| +| 0 | Gates and benchmarks: long-horizon live-parity cases and the benchmark harness (below) | — | +| 1 | S1, S2, S3 for CMA-ES and the intersection search space | 0 | +| 2 | S4 in-place writes behind snapshots, S7 interning | 0 | +| 3 | S5 MOTPE split cache, S6 TPE exact kernel, S3 for the remaining samplers (QMC) and `GroupDecomposedSearchSpace`; then the D2 measurement and question | 0, 2 | +| 4 | S10 `CachedStorage` incremental reads, S9 lazy attributes (after measuring), S8 bulk attribute API | 0 | +| 5 | S11 OpenBLAS policy (measure, then choose) | 1 | +| 6 | S12 compact finished trials | 2 | +| 7 | D1 `reuseToldOptimizer` | 1 | +| 8 | Docs: the "Performance and compatibility" page; README Performance rerun; THREADING.md §6 (the new sampler state is guarded by the sampler lock); catalog | each phase | + +**Gate A: no observable change** (every S item): +- The live-parity suites: unit, interop with the engine required, package parity. +- Phase 0 adds [Slow] interop cases that run Python Optuna and compare every trial, stored pickles included: + - CMA-ES 40-D for 3,000 trials, and the failing-trial study; + - TPE 40-D for 1,000 trials; + - MOTPE 300 trials; + - NSGA-II 2,000 trials. + Python is the oracle on every CI OS, so no hash is committed (the bit-exact suites depend on each image's libm). +- During development: a before/after hash probe of seeded studies, built from two trees. `dotnet run probe.cs` re-runs + a stale build after a referenced project changes, so build explicitly and check that the DLL holds the new code. + +**Gate B: concurrency.** +- `ConcurrencyGun` and plan 12's parallel-read tests stay green. +- Each phase that adds state to a sampler or storage adds `Fire` / `ReadInLockstep` cases for it. + +**Gate C: the gain is real.** +- The §7 workloads before and after, in Release, with power throttling opted out and the OpenBLAS thread count stated. +- Every number goes into this plan and, at the end, the README. + +**Gate D: two paths** (D items). +- Both paths are tested; the default path keeps Gate A. +- A guard test pins the default. + +## 7. Targets (to measure, not promises) + +| Workload | Today (`d0558f5`) | After the S items (expected) | Rustuna 0.2.1 | +|---|---|---|---| +| CMA-ES, 10k / 100k | 18.3 / 812 s | 2.6 / 26 s measured in the prototype with single-threaded OpenBLAS; lower after S4 and S12 | 1.6 / 13.6 s | +| NSGA-II, 10k / 100k | 3.5 / 62.9 s | about half, if S4 removes most of the 66 % | 0.8 / 19.4 s | +| MOTPE, 1k | 36.6 s | about half, from S5 (24 %) and S4 (23 %) | 6.3 s | +| TPE, 1k | 9.9 s | unknown until S6; the scoring is 58 % | 1.4 s | + +Rustuna will stay ahead on TPE, where its acquisition math is native and ours follows NumPy's order. It will also stay +ahead wherever Optuna's protocol costs time (the pickle per generation, per-trial system attributes); keeping that +protocol is what lets Python workers share OptunaSharp's studies. + +## 8. Risks + +- **S4 aliasing.** A `FrozenTrial` handed out and then changed would silently break samplers and users. Tests must + cover every path that hands out a snapshot, under concurrency. +- **Index memory.** The folds and caches hold references the storage already holds; they must reset on a new study and + must not pin discarded or deleted studies. +- **Database storages.** S1-S3 save sampler work there too, but the per-ask history read stays O(n) until S10. +- **Two paths rot.** Every D item doubles a code path; rule 4 builds one only with a measured gain, and Gate D keeps + both tested. +- **Stale measurements.** Throttling (EcoQoS), Debug builds of file-based probes and stale probe builds have each + produced wrong numbers in this repository. Gate C's conditions exist because of them. + +## 9. Open questions for the owner + +- D3 (trial queue) and D4 (discarding) after S12's memory numbers. +- D2's default once S6 and its fast variant are measured. +- §5.5's switch shape for future switches; D1 uses a constructor parameter. diff --git a/docs/plans/README.md b/docs/plans/README.md index 80a9a7c..dcf7be2 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -46,6 +46,7 @@ numeric prefix is the recommended execution order. | [10](10-threading.md) | Threading: a configurable `nJobs` executor, then seven follow-ups | the reused worker threads of `9cfcb12` (done) | N | ★★ / medium | | [11](11-threading-round-2.md) | Threading, round 2: sampler per worker, OpenBLAS threads, locked log handlers, the async question | plan 10 (done) | N | ★★ / medium | | [12](12-parallel-read-duplicates.md) | Parallel reads never hand one point to two trials: per-sampler fixes and tests, the bridges pinned | plan 11 (done) | N | ★★ / medium | +| [13](13-rustuna-derived-performance.md) | Rustuna-derived performance: compatible speedups as the only path, opt-in switches where results would change | plans 10-12 (done) | N | ★★★ / high | **Threading workstream (10, 11, 12).** Unrelated to JANUS. Plan 12 (done 2026-11-08) makes every Core sampler give trials that ask at once different points, without changing sequential runs, and pins what the four Python bridges @@ -57,6 +58,14 @@ through an options class at four levels. Phase 2, done only after Phase 1, close [`docs/THREADING.md`](../THREADING.md) §12: OS seed width, the terminator lock, Redis round trips and `DedicatedThreads`, cancellation, `PymooSampler` on any thread, and ML.NET search under `nJobs`. +**Performance workstream (13).** Designed with the owner on 2026-11-08, not started. Rustuna, the Optuna team's +Rust port, runs its announcement's workloads 3-60x faster than OptunaSharp. Plan 13 adopts each of its techniques that +keeps every result identical as the only implementation; a scratch prototype took CMA-ES at 100,000 trials from 812 s +to 26 s that way, byte-identical. The techniques that would change results are measured, and the owner decides each +one: CMA-ES reusing its told optimizer is opt-in with Optuna's path as the default; TPE fast numerics waits for the exact +fused kernel; the trial queue and trial discarding are deferred, with compact storage of finished trials proposed +instead of discarding. + **Tier-X pymoo-interop workstream (08–09).** Separate from the Tier-N native ports above: unlocked by the 2026-09-10 pymoo-on-numpy-2.4.2 migration (pymoo now runs in the interop interpreter, commit `9c7aec2`), these ship a `OptunaSharp.Interop.Pymoo` satellite that drives the **real** pymoo via pythonnet (the From 99e9e738a0062f418024ea9eb452d8ff2df7929b Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 23:17:16 +0200 Subject: [PATCH 009/145] perf(cmaes): restore the optimizer once per generation and fold the history incrementally, same results as Optuna; opt-in reuseToldOptimizer Plan 13 (docs/plans/13-rustuna-derived-performance.md), phase 1 (items S1-S3) and item D1. Optuna's CmaEsSampler restores its optimizer from the pickled state and rescans every trial on every ask, which makes a run quadratic in its trial count (40-D, seed 0: 18.3 s for 10k trials, 812 s for 100k; Rustuna 0.2.1 takes 1.6 s and 13.6 s). The scratch prototype of session 5417a82e took 100k trials to about 26 s with byte-identical results; this is that prototype, rewritten for production. What changed, none of it observable: - Samplers/CmaEs/CmaTrialHistory.cs (new): Optuna's per-ask _get_trials, _restore_optimizer backward scan and _get_solution_trials, kept up to date incrementally. Finished trials cannot change, so the leading run of finished trials is folded once, in trial order, into a completed count, a by-generation index and the latest state holder. Every refresh classifies only the trials after the first unfinished one, which keeps Optuna's scan order when trials finish out of order. A history that is not the one folded (another storage, study, or trials at the folded positions) starts over and tells the owner, so the sampler drops anything keyed by trial id. - CmaEsSampler: the optimizer deserialized from the latest stored state is cached under the id of the trial holding that state and reused by the generation's remaining asks, eigendecomposition included. It is dropped after every tell: Optuna's pickle stores only C's upper triangle and a restore mirrors it, while the told C keeps the update's floating-point asymmetry. Measured in the prototype: keeping it moved seeded 40-D results from generation 3. GetTrials/GetSolutionTrials are replaced by the history; the per-trial filter is AsSamplerTrial, so a pruned trial's value-rewritten copy is made once when folded. SampleIndependent's warning check reads the history too. - BaseStorage.HasZeroCopyTrialsView (internal, true for InMemoryStorage) and Study.GetTrialsForSampler: samplers that read the history every ask use the zero-copy view where it is one, and Optuna's read (including the per-trial cache) elsewhere, so a database-backed storage makes no extra round trip. - IntersectionSearchSpace.Calculate reads through GetTrialsForSampler: its walk already stopped at the cached trial number; the full copy of the history was what grew with the study. D1, opt-in (owner's decision: Optuna's path stays the default): CmaEsSampler(..., reuseToldOptimizer: false). When true, the optimizer that just told a generation keeps answering the asks after its trial completes, as Rustuna keeps its live optimizer: about 15 % faster on a 40-D study, and seeded runs stop matching Python Optuna from about the third generation (same quality: 40-D sphere, 10 seeds, best-value ratio median 0.79, range 0.51-1.48). The state is still stored every generation exactly as Optuna stores it, so resuming and Python workers keep working. The parameter comes after every Optuna parameter. Tests: - CmaTrialHistoryTests: randomly evolving studies (600 steps, 5 seeds; out-of-order completion, failures, pruned trials with and without intermediate values, attributes written while running) compared at every step with a naive re-implementation of Optuna's scans; a stuck running trial; the pruned copy made once; reset on another storage, study, folded trial or a shorter history; a holder in the tail; null arguments. - CmaEsSamplerCacheTests: one restore per tell in a sequential run (RestoreCount, internal); a failed teller's generation told again from the previous state, the stored state identical byte for byte; a second study on another in-memory storage (colliding trial ids) never reuses the first study's optimizer; GetTrialsForSampler is the zero-copy view in memory and Optuna's read on a journal file; 8 parallel workers, shared and per-worker. D1: off by default (identical trials, one restore per tell), one restore per run when on, a failed teller restores the previous state, a default sampler resumes a study the reusing sampler wrote, and both paths reach 1e-6 on a 10-D sphere for three seeds. - Interop, live Python: long runs (10-D 400 trials, 6-D 300), 40-D 1,200 trials ([Slow]), failing tellers (every other teller fails, both languages failing the same trials), consider_pruned_trials, SepCMA and lr-adapt over 30+ generations: every trial and every stored pickle equal Optuna's. The mixed C#/Python fleet with the C# worker reusing its told optimizer completes with the reference generations. Results: the CI unit filter (OpenBugs/HighMemory/RequiresPython/Slow excluded) 1998/1998 on net8.0 and on net10.0; unit CMA-ES/pickle/search-space 103 pass with no filter (the 3 failures are known [OpenBugs]); concurrency suites 243/243; live-Python CMA-ES suite 41/41 (engine required); documentation gate 0 issues. Phase 2 of the plan (in-place writes to a running trial nobody holds, S4; shared distributions and parameter names, S7) follows in its own commit; InMemoryStorage.cs carries only the HasZeroCopyTrialsView override here. --- .../Samplers/CmaEs/CmaTrialHistory.cs | 221 ++++++++++++ src/OptunaSharp.Core/Samplers/CmaEsSampler.cs | 227 +++++++++---- .../SearchSpace/IntersectionSearchSpace.cs | 12 +- src/OptunaSharp.Core/Storages/BaseStorage.cs | 21 ++ .../Storages/InMemoryStorage.cs | 7 + src/OptunaSharp.Core/Study.cs | 27 ++ .../CmaEsMixedFleetInteropTests.cs | 27 +- .../CmaEsSamplerParityTests.cs | 181 ++++++++++ .../Samplers/CmaEsSamplerCacheTests.cs | 321 ++++++++++++++++++ .../Samplers/CmaTrialHistoryTests.cs | 289 ++++++++++++++++ 10 files changed, 1269 insertions(+), 64 deletions(-) create mode 100644 src/OptunaSharp.Core/Samplers/CmaEs/CmaTrialHistory.cs create mode 100644 test/OptunaSharp.Tests/Samplers/CmaEsSamplerCacheTests.cs create mode 100644 test/OptunaSharp.Tests/Samplers/CmaTrialHistoryTests.cs diff --git a/src/OptunaSharp.Core/Samplers/CmaEs/CmaTrialHistory.cs b/src/OptunaSharp.Core/Samplers/CmaEs/CmaTrialHistory.cs new file mode 100644 index 0000000..13defa8 --- /dev/null +++ b/src/OptunaSharp.Core/Samplers/CmaEs/CmaTrialHistory.cs @@ -0,0 +1,221 @@ +using System; +using System.Collections.Generic; +using OptunaSharp.Storages; + +namespace OptunaSharp.Samplers.CmaEs; + +/// +/// The parts of a study's history reads on every ask, kept up to date incrementally: +/// how many trials Optuna's _get_trials would return, the latest of them holding a stored optimizer state +/// (_restore_optimizer's backward scan), and the ones stamped with a given generation +/// (_get_solution_trials). Optuna rebuilds all three from a full copy of the history on every ask, which +/// makes a run quadratic in its trial count; this class gives the same trials in the same order at the cost of the +/// trials that changed since the last ask (Rustuna plan 13, item S2). +/// +/// +/// +/// Why the results are identical. A finished trial cannot change (storages refuse updates to it), so the +/// leading run of finished trials, in trial-number order, is folded in once: each completed one is counted, +/// filed under its generation stamp, and remembered when it holds an optimizer state. Every refresh classifies +/// only the trials after the first unfinished one (the "tail"), because those may still change. Folded trials +/// all precede the tail's, so "folded, then tail" is Optuna's scan order. A trial that finishes out of order +/// (another worker's) waits in the tail until every earlier trial has finished, which keeps that order. +/// +/// +/// Not thread-safe. Its single owner calls it inside SampleRelative / SampleIndependent, +/// which Trial runs under the sampler's lock (BaseSampler.Locked); a per-worker sampler instance +/// owns its own history. +/// +/// +/// Cost of a stuck trial. A trial that stays RUNNING (a dead worker without heartbeats) stops the fold, so +/// every later refresh rescans the tail from it: speed degrades toward Optuna's per-ask cost, results stay +/// the same. Failing stale trials (fail_stale_trials) lets the fold move on. +/// +/// +internal sealed class CmaTrialHistory +{ + /// + /// Maps a finished or running trial to the snapshot the sampler treats as completed (itself, or a pruned trial + /// rewritten with its last intermediate value), or null when the sampler ignores it. Mirrors the filter of + /// Optuna's CmaEsSampler._get_trials. + /// + private readonly Func _asSamplerTrial; + + /// The system-attribute key of the generation stamp, e.g. cma:generation. + private readonly string _generationKey; + + /// The prefix of the stored optimizer-state chunks, e.g. cma:optimizer. + private readonly string _optimizerKeyPrefix; + + /// The storage the fold describes; trial ids and positions mean nothing in another one. + private BaseStorage? _storage; + + /// The study the fold describes. + private int _studyId = -1; + + /// How many leading trials (by number) are finished and folded in. + private int _foldedCount; + + /// + /// The id of the last folded trial: a history whose trial at that position has another id is not the one + /// folded (a new study under a reused id), so the fold starts over. + /// + private int _lastFoldedTrialId = -1; + + /// How many folded trials the sampler treats as completed. + private int _foldedCompletedCount; + + /// The folded completed trials by generation stamp (-1 when unstamped), each list in trial order. + private readonly Dictionary> _foldedByGeneration = new(); + + /// The latest folded completed trial that holds an optimizer state; null when none does. + private FrozenTrial? _foldedLatestStateHolder; + + /// The completed trials after the first unfinished one, from the last refresh, in trial order. + private readonly List _tail = new(); + + /// Creates an empty history; the first folds the study. + /// + /// The sampler's completed-trial filter (see ). It is called once per trial when the + /// trial is folded, and on every refresh for the trials in the tail, so it must not depend on anything but the + /// trial; the sampler's pruned-trial copies are made once and reused for the rest of the run. + /// + /// The generation stamp's system-attribute key. + /// The optimizer-state chunks' key prefix. + /// An argument is . + internal CmaTrialHistory(Func asSamplerTrial, string generationKey, string optimizerKeyPrefix) + { + _asSamplerTrial = asSamplerTrial ?? throw new ArgumentNullException(nameof(asSamplerTrial)); + _generationKey = generationKey ?? throw new ArgumentNullException(nameof(generationKey)); + _optimizerKeyPrefix = optimizerKeyPrefix ?? throw new ArgumentNullException(nameof(optimizerKeyPrefix)); + } + + /// The number of trials Optuna's _get_trials would return at the last refresh. + internal int CompletedCount => _foldedCompletedCount + _tail.Count; + + /// + /// The latest completed trial, in trial order, that holds a stored optimizer state: the trial Optuna's + /// _restore_optimizer restores from. Null when no completed trial holds one. + /// + internal FrozenTrial? LatestStateHolder + { + get + { + // The tail's trials come after every folded one, so a holder there is the latest. + for (int i = _tail.Count - 1; i >= 0; i--) + if (HoldsOptimizerState(_tail[i])) + return _tail[i]; + return _foldedLatestStateHolder; + } + } + + /// + /// Whether the last started over (the first refresh, another storage or study, or a + /// history that no longer starts with the folded trials). The owner must then drop anything it keyed by trial + /// id, because ids from another storage can repeat. + /// + internal bool WasReset { get; private set; } + + /// + /// Brings the fold up to date with and classifies the trials after its first + /// unfinished one. + /// + /// + /// Every trial of the study in trial-number order, as Study.GetTrialsForSampler returns it. Read once per + /// position; not kept. + /// + /// The storage the history came from. + /// The study the history belongs to. + /// or is . + internal void Refresh(IReadOnlyList history, BaseStorage storage, int studyId) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentNullException.ThrowIfNull(storage); + + // The fold is valid only for the history it was built from: same storage, same study, and the folded + // positions still hold the same trials. Anything else (a new study under a reused id, a fresh storage) starts + // over; trial ids are unique within one storage, so the last folded id identifies the history. + bool sameHistory = ReferenceEquals(_storage, storage) && _studyId == studyId + && history.Count >= _foldedCount + && (_foldedCount == 0 || history[_foldedCount - 1].TrialId == _lastFoldedTrialId); + WasReset = !sameHistory; + if (!sameHistory) + { + _storage = storage; + _studyId = studyId; + _foldedCount = 0; + _lastFoldedTrialId = -1; + _foldedCompletedCount = 0; + _foldedByGeneration.Clear(); + _foldedLatestStateHolder = null; + } + + // Fold the leading finished trials. Each history read happens once; the snapshot read is the one folded. + while (_foldedCount < history.Count) + { + FrozenTrial source = history[_foldedCount]; + if (!source.State.IsFinished()) + break; + FrozenTrial? completed = _asSamplerTrial(source); + if (completed is not null) + { + _foldedCompletedCount++; + long generation = GenerationOf(completed); + if (!_foldedByGeneration.TryGetValue(generation, out List? sameGeneration)) + _foldedByGeneration[generation] = sameGeneration = new List(); + sameGeneration.Add(completed); + if (HoldsOptimizerState(completed)) + _foldedLatestStateHolder = completed; + } + _lastFoldedTrialId = source.TrialId; + _foldedCount++; + } + + // The tail: trials after the first unfinished one, classified afresh each refresh because they may change. + _tail.Clear(); + for (int i = _foldedCount; i < history.Count; i++) + { + FrozenTrial? completed = _asSamplerTrial(history[i]); + if (completed is not null) + _tail.Add(completed); + } + } + + /// + /// The completed trials stamped with , in trial order: what Optuna's + /// _get_solution_trials returns from the trials of the last refresh. + /// + /// The optimizer generation whose trials the next tell needs. + /// A new list the caller may keep; folded trials first, then the tail's. + internal List SolutionTrials(int generation) + { + var result = _foldedByGeneration.TryGetValue(generation, out List? folded) + ? new List(folded) + : new List(); + foreach (FrozenTrial trial in _tail) + if (GenerationOf(trial) == generation) + result.Add(trial); + return result; + } + + /// The trial's generation stamp, or -1 (Optuna's default) without one. + /// A trial the sampler treats as completed. + /// The stamp the sampler wrote when the trial asked. + /// The stamp is not a number (a storage corrupted by hand). + private long GenerationOf(FrozenTrial trial) + => trial.SystemAttrs.TryGetValue(_generationKey, out object? stamp) ? Convert.ToInt64(stamp) : -1; + + /// + /// Whether the trial stores an optimizer state: any system attribute under the chunk prefix, the test of + /// Optuna's _restore_optimizer. + /// + /// A trial the sampler treats as completed. + /// True when at least one optimizer-state chunk is present. + private bool HoldsOptimizerState(FrozenTrial trial) + { + foreach (string key in trial.SystemAttrs.Keys) + if (key.StartsWith(_optimizerKeyPrefix, StringComparison.Ordinal)) + return true; + return false; + } +} diff --git a/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs b/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs index 57ac735..69dfae0 100644 --- a/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs +++ b/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs @@ -28,6 +28,13 @@ namespace OptunaSharp.Samplers; /// JSON) are still read; Python cannot read those, so a study gains Python workers once a current /// C# worker has stored the next state. With with_margin=True the Python workers need /// cmaes 0.13.1, whose CMAwM state OptunaSharp reads. +/// +/// Faster than Optuna's structure, same results. Optuna restores the optimizer from its pickle +/// and rescans every trial on every ask, which makes a run quadratic in its trial count. This sampler keeps +/// the restored optimizer for the rest of its generation and folds the history incrementally +/// (CmaTrialHistory), reading an in-memory study through its zero-copy view; seeded trials and every +/// stored byte stay identical to Optuna's (Rustuna plan 13, phase 1; the measured gain is in +/// docs/plans/13-rustuna-derived-performance.md). /// /// /// @@ -60,6 +67,49 @@ public class CmaEsSampler : BaseSampler private readonly List? _sourceTrials; private readonly string _attrPrefix; + /// + /// The history this sampler reads on every ask (Optuna's _get_trials, the latest stored state, a + /// generation's trials), kept up to date incrementally instead of rebuilt from a full copy each time. Same + /// trials, same order (Rustuna plan 13, item S2). Touched only under the sampler's lock. + /// + private readonly CmaTrialHistory _history; + + /// + /// The optimizer deserialized from the latest stored state, kept for the remaining asks of its generation so + /// each ask skips the unpickling and reuses the eigendecomposition the first ask computed (plan 13, item S1). + /// Null when nothing is cached. Touched only under the sampler's lock. + /// + /// + /// It answers only for 's state, and every ask that reuses it would have + /// restored an identical object: asking caches B and D and rewrites C as B·D²·Bᵀ, which a fresh restore of the + /// same stored C recomputes to the same bits, and the per-ask reseed resets the random stream and its Gaussian + /// cache. It is dropped after every tell: the told object stands neither for the state it came from (restored + /// again if the telling trial fails) nor for the state just stored, because Optuna's pickle keeps only C's + /// upper triangle and a restore mirrors it, while the told C keeps the update's floating-point asymmetry. + /// Keeping it moves seeded results (measured: from generation 3 on a 40-D sphere). + /// + private ICmaOptimizer? _cachedOptimizer; + + /// + /// The trial whose stored state was restored from; -1 when nothing is cached. + /// A holder is a finished trial, whose system attributes cannot change, so its id names its state; a state + /// another worker (C# or Python) stores lands on another trial and misses. + /// + private int _cachedHolderTrialId = -1; + + /// + /// How many stored optimizer states this instance deserialized. Lets the tests assert the cache's contract + /// (one restore per generation in a sequential run) without timing anything. + /// + internal int RestoreCount { get; private set; } + + /// + /// The opt-in reuseToldOptimizer (Rustuna plan 13, item D1; the owner chose opt-in with Optuna's path as + /// the default): after a tell, the told optimizer becomes the cache for the state it just stored instead of + /// being dropped. Faster, and no longer Optuna's samples (see for why). + /// + private readonly bool _reuseToldOptimizer; + /// /// Initial parameter values for CMA-ES (external representation). By default the mean of the /// low and high of each distribution. Deprecated in v4.9.0. @@ -79,6 +129,15 @@ public class CmaEsSampler : BaseSampler /// Use CMA-ES with margin for discrete distributions (experimental). /// Use CMA-ES with learning-rate adaptation (experimental). /// Warm Starting CMA-ES source trials (experimental). + /// + /// Off by default; turning it on makes seeded runs stop matching Python Optuna (from about the third + /// generation; same solution quality, a different run). When on, the optimizer that just told a generation keeps + /// answering the asks that follow, as Rustuna keeps its live optimizer, instead of being restored from the state + /// it stored: about 15 % faster on a 40-D study. The state is still stored every generation exactly as Optuna + /// stores it, so resuming a study and Python workers on the same storage keep working; they just do not + /// reproduce this worker's samples. An OptunaSharp extension with no Optuna counterpart, so it comes after every + /// Optuna parameter. + /// /// Configures covariance-matrix adaptation and its independent startup and unsupported-parameter fallback. /// Source-trial initialization, explicit initial parameters, separable adaptation, margin correction, or learning-rate adaptation are combined incompatibly. public CmaEsSampler( @@ -95,7 +154,8 @@ public CmaEsSampler( bool useSeparableCma = false, bool withMargin = false, bool lrAdapt = false, - List? sourceTrials = null) + List? sourceTrials = null, + bool reuseToldOptimizer = false) { if (restartStrategy != null || incPopsize != -1) { @@ -125,8 +185,10 @@ public CmaEsSampler( _withMargin = withMargin; _lrAdapt = lrAdapt; _sourceTrials = sourceTrials; + _reuseToldOptimizer = reuseToldOptimizer; _attrPrefix = useSeparableCma ? "sepcma:" : withMargin ? "cmawm:" : "cma:"; + _history = new CmaTrialHistory(AsSamplerTrial, AttrKeyGeneration, AttrKeyOptimizer); // Optuna raises an ExperimentalWarning (warn_experimental_argument) for consider_pruned_trials, // use_separable_cma, source_trials, with_margin and lr_adapt. OptunaSharp targets the Optuna 5.x @@ -176,6 +238,9 @@ private CmaEsSampler(CmaEsSampler prototype, BaseSampler independentSampler) _lrAdapt = prototype._lrAdapt; _sourceTrials = prototype._sourceTrials; _attrPrefix = prototype._attrPrefix; + _reuseToldOptimizer = prototype._reuseToldOptimizer; + // A history and an optimizer cache of its own: each worker folds the shared study by itself. + _history = new CmaTrialHistory(AsSamplerTrial, AttrKeyGeneration, AttrKeyOptimizer); } /// @@ -233,14 +298,16 @@ public override OrderedDict InferRelativeSearchSpace(S if (searchSpace.Count == 0) return new OrderedDict(); - var completedTrials = GetTrials(study); - if (completedTrials.Count < _nStartupTrials) + // Optuna's `completed_trials = self._get_trials(study)`, read incrementally: the history folds each finished + // trial once instead of copying and filtering every trial on every ask (CmaTrialHistory). + RefreshHistory(study); + if (_history.CompletedCount < _nStartupTrials) return new OrderedDict(); // When `with_margin=true`, bounds in discrete dimensions are handled inside `CMAwM`. var trans = new SearchSpaceTransform(searchSpace, transformStep: !_withMargin, transform01: true); - var restoredOptimizer = RestoreOptimizer(completedTrials); + var restoredOptimizer = RestoreOptimizer(); var optimizer = restoredOptimizer ?? InitOptimizer(trans, study.Direction, searchSpace); int nDim = (int)trans.Bounds.shape[0]; @@ -257,7 +324,8 @@ public override OrderedDict InferRelativeSearchSpace(S return new OrderedDict(); } - var solutionTrials = GetSolutionTrials(completedTrials, optimizer.Generation); + // Optuna's `_get_solution_trials(completed_trials, optimizer.generation)`, from the generation index. + var solutionTrials = _history.SolutionTrials(optimizer.Generation); if (solutionTrials.Count >= optimizer.PopulationSize) { @@ -280,6 +348,24 @@ public override OrderedDict InferRelativeSearchSpace(S string optimizerStr = SerializeOptimizer(optimizer, restored: restoredOptimizer is not null); foreach (var (key, value) in SplitOptimizerStr(optimizerStr)) study.Storage.SetTrialSystemAttr(trial.TrialId, key, value); + + if (_reuseToldOptimizer && restoredOptimizer is not null) + { + // Opt-in Rustuna path: the told object answers for the state this trial just stored, from the moment + // the trial completes and becomes the latest state holder. If it fails instead, the previous holder is + // the latest again, its id misses the cache, and that state is restored, as Optuna does. A fresh + // optimizer (no state was restored before this first tell) is not kept, so the cached object always + // carries the restored layout CmaPickle writes for every later tell. + _cachedOptimizer = optimizer; + _cachedHolderTrialId = trial.TrialId; + } + else + { + // The told object no longer equals any stored state (see _cachedOptimizer), so the next ask restores the + // state just stored, as Optuna does: one restore per generation instead of one per ask. + _cachedOptimizer = null; + _cachedHolderTrialId = -1; + } } // Caution: optimizer should update its seed value. @@ -379,25 +465,57 @@ private static ICmaOptimizer DeserializeOptimizer(string optimizerStr) }; } - /// Finds the newest stored optimizer snapshot in completed trial history. - /// Completed trial snapshots used as historical observations. - /// The newest available optimizer, or null when no stored snapshot exists. - private ICmaOptimizer? RestoreOptimizer(List completedTrials) + /// + /// Optuna's _restore_optimizer: the optimizer stored on the latest completed trial that holds one, from + /// when that trial's state is the one cached. + /// + /// The newest available optimizer, or null when no completed trial stores one. + /// The stored state is neither a legacy JSON state nor a readable cmaes pickle. + /// A legacy JSON payload is not valid JSON for its state type. + /// A legacy JSON state contains invalid optimizer constructor settings. + /// + /// The holder is , the trial Optuna's backward scan over the + /// completed trials stops at. A cache hit returns the object an earlier ask of this generation restored and asked + /// with, which produces the same samples as a fresh restore (see ). + /// + private ICmaOptimizer? RestoreOptimizer() { + FrozenTrial? holder = _history.LatestStateHolder; + if (holder is null) + return null; + if (_cachedOptimizer is not null && _cachedHolderTrialId == holder.TrialId) + return _cachedOptimizer; + // Restore a previous CMA object. - for (int idx = completedTrials.Count - 1; idx >= 0; idx--) - { - var trial = completedTrials[idx]; - var optimizerAttrs = trial.SystemAttrs - .Where(kv => kv.Key.StartsWith(AttrKeyOptimizer, StringComparison.Ordinal)) - .ToDictionary(kv => kv.Key, kv => (string)kv.Value!); - if (optimizerAttrs.Count == 0) - continue; + var optimizerAttrs = holder.SystemAttrs + .Where(kv => kv.Key.StartsWith(AttrKeyOptimizer, StringComparison.Ordinal)) + .ToDictionary(kv => kv.Key, kv => (string)kv.Value!); + var restored = DeserializeOptimizer(ConcatOptimizerAttrs(optimizerAttrs)); + RestoreCount++; + _cachedOptimizer = restored; + _cachedHolderTrialId = holder.TrialId; + return restored; + } - string optimizerStr = ConcatOptimizerAttrs(optimizerAttrs); - return DeserializeOptimizer(optimizerStr); + /// + /// Brings up to date with the study, and forgets the optimizer cache when the history + /// started over (another study or storage, whose trial ids may repeat this one's). + /// + /// The study being sampled. + /// The study no longer exists in storage. + /// + /// The read is Study.GetTrialsForSampler(useCache: true): the zero-copy view on an in-memory storage, + /// otherwise the per-trial cached list Optuna's _get_trials(deepcopy=False, use_cache=True) reads, so a + /// database-backed storage makes no more round trips than before. + /// + private void RefreshHistory(Study study) + { + _history.Refresh(study.GetTrialsForSampler(useCache: true), study.Storage, study.StudyId); + if (_history.WasReset) + { + _cachedOptimizer = null; + _cachedHolderTrialId = -1; } - return null; } /// Creates the initial optimizer in the transformed numerical box. @@ -577,8 +695,10 @@ private static bool IsCompatibleSearchSpace( if (_warnIndependentSampling) { - var completeTrials = GetTrials(study); - if (completeTrials.Count >= _nStartupTrials) + // Optuna's `len(self._get_trials(study)) >= self._n_startup_trials`, from the incremental history, so a + // categorical parameter sampled every trial no longer costs a full copy of the history each time. + RefreshHistory(study); + if (_history.CompletedCount >= _nStartupTrials) LogIndependentSampling(trial, paramName); } @@ -599,48 +719,33 @@ private void LogIndependentSampling(FrozenTrial trial, string paramName) "dynamic search space and `CategoricalDistribution` are not supported by `CmaEsSampler`")); } - /// Collects historical trials eligible for this sampler, applying its completed/pruned-trial policy. - /// Study supplying trial history, objective directions, and storage. - /// Trial snapshots eligible for the sampler history. - private List GetTrials(Study study) - { - var completeTrials = new List(); - foreach (var t in study.GetTrialsInternal(deepcopy: false, states: null, useCache: true)) - { - if (t.State == TrialState.Complete) - { - completeTrials.Add(t); - } - else if (t.State == TrialState.Pruned - && t.IntermediateValues.Count > 0 - && _considerPrunedTrials) - { - // max(t.intermediate_values.items()) — the value at the largest step key. - long maxStep = t.IntermediateValues.Keys.Max(); - double value = t.IntermediateValues[maxStep]; - // We rewrite the value of the trial `t` for sampling, so we need a copy. - var copied = t.DeepCopy(); - copied.Value = value; - completeTrials.Add(copied); - } - } - return completeTrials; - } - - /// Selects trials belonging to one optimizer generation so adaptation uses a coherent population. - /// Trial history available to the operation, in its existing order. - /// Zero-based generation whose individuals or optimizer state are requested. - /// Trials from the requested optimizer generation. - private List GetSolutionTrials(List trials, int generation) + /// + /// The per-trial filter of Optuna's _get_trials: a COMPLETE trial as it is; with + /// consider_pruned_trials, a PRUNED trial that reported values, copied with its value rewritten to the + /// value at its largest step; anything else is ignored. + /// + /// A trial snapshot from the history. + /// The snapshot the sampler treats as completed, or null when it ignores the trial. + /// + /// calls this once per trial when the trial is folded and keeps the result, so a + /// pruned trial's copy is made once per run rather than once per ask; a finished trial cannot change, so the copy + /// stays exact. + /// + private FrozenTrial? AsSamplerTrial(FrozenTrial t) { - var result = new List(); - foreach (var t in trials) + if (t.State == TrialState.Complete) + return t; + if (t.State == TrialState.Pruned && t.IntermediateValues.Count > 0 && _considerPrunedTrials) { - long g = t.SystemAttrs.TryGetValue(AttrKeyGeneration, out var v) ? Convert.ToInt64(v) : -1; - if (generation == g) - result.Add(t); + // max(t.intermediate_values.items()) — the value at the largest step key. + long maxStep = t.IntermediateValues.Keys.Max(); + double value = t.IntermediateValues[maxStep]; + // We rewrite the value of the trial `t` for sampling, so we need a copy. + var copied = t.DeepCopy(); + copied.Value = value; + return copied; } - return result; + return null; } /// diff --git a/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs b/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs index 599f491..661038d 100644 --- a/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs +++ b/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs @@ -150,7 +150,15 @@ public IntersectionSearchSpace(bool includePruned = false) /// parameter names. /// /// This cache is reused with a different study identifier. - /// The returned dictionary is a new sorted container. Distribution objects are shared with trial snapshots and are treated as immutable. + /// + /// The returned dictionary is a new sorted container. Distribution objects are shared with trial snapshots and are treated as immutable. + /// + /// The trials are read through Study.GetTrialsForSampler: on an in-memory storage that is the + /// zero-copy view, so a call costs only the trials since the cached trial number, where Optuna copies the + /// whole history first. The walk below already stops at the cached number; the copy was the part that grew + /// with the study. Elsewhere the read is Optuna's, cache included. + /// + /// public OrderedDict Calculate(Study study, bool useCache = false) { if (_studyId is null) @@ -166,7 +174,7 @@ public OrderedDict Calculate(Study study, bool useCach } (_searchSpace, _cachedTrialNumber) = SearchSpaceFunctions.Calculate( - study.GetTrialsInternal(deepcopy: false, useCache: useCache), + study.GetTrialsForSampler(useCache), _includePruned, _searchSpace, _cachedTrialNumber); diff --git a/src/OptunaSharp.Core/Storages/BaseStorage.cs b/src/OptunaSharp.Core/Storages/BaseStorage.cs index 9de30ca..70046b7 100644 --- a/src/OptunaSharp.Core/Storages/BaseStorage.cs +++ b/src/OptunaSharp.Core/Storages/BaseStorage.cs @@ -304,6 +304,27 @@ public abstract List GetAllTrials( public virtual IReadOnlyList GetTrialsPrefixView(int studyId) => GetAllTrials(studyId, deepcopy: false); + /// + /// Whether really is O(1) here (a window over a live list), rather than + /// the base fallback that copies every trial. Samplers that read the history on every ask switch to the view + /// only when this is . + /// + /// + /// + /// Why a flag rather than "always the view": the fallback is a fresh GetAllTrials, which on a + /// database-backed storage is a synchronization with the database. A sampler that used to share the + /// per-trial cached list (Study.GetTrialsInternal(useCache: true)) would issue one more query per + /// ask if it read the fallback view instead. So such storages keep their current read path, and only the + /// storages that override this to get the zero-copy one. + /// + /// + /// Internal on purpose: a third-party storage cannot opt in, because the samplers then read its view + /// without the storage's lock, which is safe only for the in-memory list semantics documented on + /// . + /// + /// + internal virtual bool HasZeroCopyTrialsView => false; + /// Count the number of trials in a study. /// If no study with the matching ID exists. /// Storage identifier of the study; distinct from its name. diff --git a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs index b4fa39f..fb335cc 100644 --- a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs +++ b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs @@ -521,6 +521,13 @@ public override IReadOnlyList GetTrialsPrefixView(int studyId) } } + /// + /// + /// : above wraps the live list without copying it, so + /// samplers that read the history on every ask can afford to read it through the view. + /// + internal override bool HasZeroCopyTrialsView => true; + /// Storage identifier of the study; distinct from its name. /// Whether to copy trial records; treat records returned with false as read-only. /// States to include; null includes every state. diff --git a/src/OptunaSharp.Core/Study.cs b/src/OptunaSharp.Core/Study.cs index 99e2a4f..53a7d4f 100644 --- a/src/OptunaSharp.Core/Study.cs +++ b/src/OptunaSharp.Core/Study.cs @@ -279,6 +279,33 @@ internal List GetTrialsInternal( /// The study no longer exists in storage. internal IReadOnlyList GetTrialsView() => Storage.GetTrialsPrefixView(StudyId); + /// + /// Every trial of this study, for a sampler that reads the history on every ask: the zero-copy view where the + /// storage has one, otherwise exactly the read Optuna's samplers make + /// (study._get_trials(deepcopy=False, use_cache=...)). + /// + /// + /// For storages without a zero-copy view: whether to share the per-thread snapshot that lives until the next + /// ask (Optuna's use_cache). Ignored where the view is used, since the view costs nothing to take again. + /// + /// Trials in ascending trial-number order; read-only. + /// The study no longer exists in storage. + /// + /// + /// Same trials, same order: the two reads differ only in how fresh each snapshot is. The view returns each + /// trial's current snapshot, a copied list the one current when it was taken. In a run with one trial at + /// a time nothing changes in between, so seeded results are identical (Rustuna plan 13, phase 1). + /// + /// + /// The fallback keeps database-backed storages on their existing read, so switching a sampler to this + /// method never adds a database round trip (). + /// + /// + internal IReadOnlyList GetTrialsForSampler(bool useCache) + => Storage.HasZeroCopyTrialsView + ? Storage.GetTrialsPrefixView(StudyId) + : GetTrialsInternal(deepcopy: false, states: null, useCache: useCache); + /// Python Study._get_best_trial(deepcopy). /// Whether to copy trial records; treat records returned with false as read-only. /// The best feasible completed single-objective trial, copied when requested. diff --git a/test/OptunaSharp.Tests.Interop/CmaEsMixedFleetInteropTests.cs b/test/OptunaSharp.Tests.Interop/CmaEsMixedFleetInteropTests.cs index 8169be8..c7b2496 100644 --- a/test/OptunaSharp.Tests.Interop/CmaEsMixedFleetInteropTests.cs +++ b/test/OptunaSharp.Tests.Interop/CmaEsMixedFleetInteropTests.cs @@ -115,6 +115,30 @@ public void CSharpAndPythonWorkers_TakingTurns_WithMargin_CompleteTheStudy(bool } } + /// + /// The C# seat opts into reuseToldOptimizer (Rustuna plan 13, D1): its samples are no longer Optuna's, but + /// it still stores Optuna's state every generation, so the Python seat keeps restoring and continuing it. Every + /// trial completes, the states change hands both ways, and the study passes through the same generations at the + /// same trials as the Python-only fleet. + /// + /// Whether the C# worker takes the first turn. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(true)] + [DataRow(false)] + public void CSharpWorkerReusingItsToldOptimizer_AndAPythonWorker_CompleteTheStudy(bool csharpFirst) + { + PythonSession.EnsureOrInconclusive(); + + var reference = RunFleet("cma", "reference", firstIsCSharp: false, secondIsCSharp: false); + var mixed = RunFleet("cma_reuse", "mixed", firstIsCSharp: csharpFirst, secondIsCSharp: !csharpFirst); + + mixed.Should().HaveCount(Turns * TrialsPerTurn).And.OnlyContain(t => t.State == TrialState.Complete); + AssertStatesChangedHands(mixed, "cma:"); + mixed.Select(t => Generation(t, "cma:")).Should().Equal(reference.Select(t => Generation(t, "cma:")), + "tells happen when a generation's trials are complete, whatever their values"); + } + /// /// Runs a two-seat fleet on a new journal: the first seat (seed 11) takes the even turns, the second (seed 22) /// the odd ones, each seat a C# or a Python worker whose study, storage and sampler live across its turns. @@ -200,6 +224,7 @@ private IReadOnlyList RunFleet(string variant, string name, bool fi private static CmaEsSampler CsSampler(string variant, int seed) => variant switch { "cma" => new CmaEsSampler(seed: seed), + "cma_reuse" => new CmaEsSampler(seed: seed, reuseToldOptimizer: true), "sepcma" => new CmaEsSampler(seed: seed, useSeparableCma: true), "lr_adapt" => new CmaEsSampler(seed: seed, lrAdapt: true), "with_margin" => new CmaEsSampler(seed: seed, withMargin: true), @@ -212,7 +237,7 @@ private IReadOnlyList RunFleet(string variant, string name, bool fi /// The variant is unknown. private static string PySampler(string variant) => variant switch { - "cma" => "optuna.samplers.CmaEsSampler(seed=seed)", + "cma" or "cma_reuse" => "optuna.samplers.CmaEsSampler(seed=seed)", "sepcma" => "optuna.samplers.CmaEsSampler(seed=seed, use_separable_cma=True)", "lr_adapt" => "optuna.samplers.CmaEsSampler(seed=seed, lr_adapt=True)", "with_margin" => "optuna.samplers.CmaEsSampler(seed=seed, with_margin=True)", diff --git a/test/OptunaSharp.Tests.Interop/CmaEsSamplerParityTests.cs b/test/OptunaSharp.Tests.Interop/CmaEsSamplerParityTests.cs index 2471c75..bd99f95 100644 --- a/test/OptunaSharp.Tests.Interop/CmaEsSamplerParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/CmaEsSamplerParityTests.cs @@ -396,6 +396,187 @@ public void WithMargin_Trajectory_MatchesLiveOptuna(int seed, int nTrials) $"study.optimize(_obj, n_trials={nTrials})\n"); } + // ---- Rustuna plan 13, phase 1: the optimizer cache and the incremental history must leave every trial and every + // ---- stored optimizer state identical to Optuna's over many generations, failing tellers and pruned trials. + + /// + /// 30-40 generations of the plain sampler: the optimizer is restored once per generation and reused, and the + /// history is folded incrementally, yet every trial and stored state equals live Optuna's. + /// + /// Search-space dimension. + /// Sampler seed. + /// Number of trials. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(10, 3, 400)] + [DataRow(6, 11, 300)] + public void LongRun_FloatSphere_FullState_IsBitIdentical(int dim, int seed, int nTrials) + { + PythonSession.EnsureBitExactHostOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "cma", + sampler: new CmaEsSampler(seed: seed)); + csStudy.Optimize(SphereCs(dim), nTrials: nTrials); + + AssertParityExceptOptimizerBlob(csStudy, + $"study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed={seed}))\n" + + SpherePy(dim) + + $"study.optimize(_obj, n_trials={nTrials})\n"); + } + + /// + /// The 40-D workload of the Rustuna comparison for 1,200 trials (80 generations), where a wrong cache hit or a + /// mis-ordered generation would show within a few generations. + /// + [TestMethod] + [TestCategory("RequiresPython")] + [TestCategory("Slow")] + public void LongRun_40D_FullState_IsBitIdentical() + { + PythonSession.EnsureBitExactHostOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "cma", + sampler: new CmaEsSampler(seed: 0)); + csStudy.Optimize(SphereCs(40), nTrials: 1200); + + AssertParityExceptOptimizerBlob(csStudy, + "study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed=0))\n" + + SpherePy(40) + + "study.optimize(_obj, n_trials=1200)\n"); + } + + /// + /// Every other trial that told a generation (its sampling stored a state) fails afterwards, plus every 7th + /// trial. A failed teller never becomes the latest state holder, so the next ask must restore the previous state + /// (the cache was dropped at the tell) and tell the same generation again, exactly as Optuna does. + /// + /// + /// Thrown by the objective on purpose; Optimize's catch turns it into a FAIL trial, so it never + /// leaves the test. + /// + [TestMethod] + [TestCategory("RequiresPython")] + public void FailingTellers_FullState_IsBitIdentical() + { + PythonSession.EnsureBitExactHostOrInconclusive(); + + var sphere = SphereCs(6); + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "cma", + sampler: new CmaEsSampler(seed: 5)); + int tellersSeen = 0; + csStudy.Optimize(t => + { + double v = sphere(t); + bool told = csStudy.Storage.GetTrialSystemAttrs(t.TrialId).ContainsKey("cma:optimizer:0"); + if ((told && tellersSeen++ % 2 == 0) || t.Number % 7 == 3) + throw new InvalidOperationException("planned failure"); + return v; + }, nTrials: 300, @catch: new[] { typeof(InvalidOperationException) }); + + csStudy.Trials.Count(t => t.State == TrialState.Fail + && t.SystemAttrs.ContainsKey("cma:optimizer:0")).Should().BeGreaterThan(5, "failed tellers must occur"); + AssertParityExceptOptimizerBlob(csStudy, + "study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed=5))\n" + + SpherePy(6) + + "_tellers_seen = [0]\n" + + "def _obj_failing(t):\n" + + " v = _obj(t)\n" + + " told = 'cma:optimizer:0' in t.study._storage.get_trial_system_attrs(t._trial_id)\n" + + " fail = False\n" + + " if told:\n" + + " fail = _tellers_seen[0] % 2 == 0\n" + + " _tellers_seen[0] += 1\n" + + " if fail or t.number % 7 == 3:\n" + + " raise ValueError('planned failure')\n" + + " return v\n" + + "study.optimize(_obj_failing, n_trials=300, catch=(ValueError,))\n"); + } + + /// + /// With consider_pruned_trials, pruned trials join the history as copies carrying their last + /// intermediate value; the history makes each copy once, when the trial is folded, and the run still equals + /// Optuna's, which copies them on every ask. + /// + /// + /// Thrown by the objective on purpose for every 5th trial; the study records it as a PRUNED trial, so it never + /// leaves the test. + /// + [TestMethod] + [TestCategory("RequiresPython")] + public void ConsiderPrunedTrials_FullState_IsBitIdentical() + { + PythonSession.EnsureBitExactHostOrInconclusive(); + + var sphere = SphereCs(5); + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "cma", + sampler: new CmaEsSampler(seed: 2, considerPrunedTrials: true)); + csStudy.Optimize(t => + { + double v = sphere(t); + if (t.Number % 5 == 2) + { + t.Report(v + 1.0, 1); + t.Report(v * 0.5, 3); + throw new TrialPruned(); + } + return v; + }, nTrials: 250); + + csStudy.Trials.Count(t => t.State == TrialState.Pruned).Should().BeGreaterThan(40); + AssertParityExceptOptimizerBlob(csStudy, + "study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed=2, consider_pruned_trials=True))\n" + + SpherePy(5) + + "def _obj_pruning(t):\n" + + " v = _obj(t)\n" + + " if t.number % 5 == 2:\n" + + " t.report(v + 1.0, 1)\n" + + " t.report(v * 0.5, 3)\n" + + " raise optuna.TrialPruned()\n" + + " return v\n" + + "study.optimize(_obj_pruning, n_trials=250)\n"); + } + + /// The separable variant over 30+ generations: its own state prefix (sepcma:) and diagonal state. + [TestMethod] + [TestCategory("RequiresPython")] + public void SeparableCma_LongRun_FullState_IsBitIdentical() + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "cma", + sampler: new CmaEsSampler(seed: 4, useSeparableCma: true)); + csStudy.Optimize(SphereCs(8), nTrials: 300); + + AssertParityExceptOptimizerBlob(csStudy, + "import warnings; warnings.simplefilter('ignore')\n" + + "study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed=4, use_separable_cma=True))\n" + + SpherePy(8) + + "study.optimize(_obj, n_trials=300)\n"); + } + + /// Learning-rate adaptation over 30+ generations: its extra state (the lr-adapt moments) survives the cache. + [TestMethod] + [TestCategory("RequiresPython")] + public void LrAdapt_LongRun_FullState_IsBitIdentical() + { + PythonSession.EnsureBitExactHostOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "cma", + sampler: new CmaEsSampler(seed: 9, lrAdapt: true)); + csStudy.Optimize(SphereCs(6), nTrials: 300); + + AssertParityExceptOptimizerBlob(csStudy, + "study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed=9, lr_adapt=True))\n" + + SpherePy(6) + + "study.optimize(_obj, n_trials=300)\n"); + } + // Deterministic shifted sphere with an explicit left-to-right sum (NOT Python's builtin sum(), // which since 3.12 uses Neumaier compensated summation and would drift ~1 ULP from the C# sum). private static Func SphereCs(int dim) diff --git a/test/OptunaSharp.Tests/Samplers/CmaEsSamplerCacheTests.cs b/test/OptunaSharp.Tests/Samplers/CmaEsSamplerCacheTests.cs new file mode 100644 index 0000000..774a84a --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/CmaEsSamplerCacheTests.cs @@ -0,0 +1,321 @@ +using System; +using System.IO; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; +using OptunaSharp.Storages.Journal; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// The optimizer cache and incremental history of (Rustuna plan 13, phase 1), tested +/// through the sampler. That the trials themselves equal Optuna's is the live interop suite's job +/// (CmaEsSamplerParityTests); these pin the cache's contract (one restore per generation, a failed teller's +/// state restored again, nothing reused across storages) and the history read +/// (Study.GetTrialsForSampler). +/// +[TestClass] +public class CmaEsSamplerCacheTests +{ + /// A shifted sphere over float parameters in [-5, 5]. + /// Number of parameters. + /// Where the optimum is, on every axis. + /// The objective. + private static Func Sphere(int dim, double shift) => t => + { + double v = 0; + for (int i = 0; i < dim; i++) + { + double x = t.SuggestFloat($"x{i}", -5, 5) - shift; + v += x * x; + } + return v; + }; + + /// Whether the trial stored an optimizer state, i.e. it told a generation. + /// A finished trial. + /// True when a cma:optimizer:* chunk is present. + private static bool HoldsState(FrozenTrial trial) + => trial.SystemAttrs.Keys.Any(k => k.StartsWith("cma:optimizer:", StringComparison.Ordinal)); + + /// The trial's stored optimizer state, chunks concatenated in order. + /// A trial that told a generation. + /// The stored hex pickle. + private static string StoredState(FrozenTrial trial) + => string.Concat(trial.SystemAttrs.Keys.Where(k => k.StartsWith("cma:optimizer:", StringComparison.Ordinal)) + .OrderBy(k => int.Parse(k["cma:optimizer:".Length..])) + .Select(k => (string)trial.SystemAttrs[k]!)); + + /// + /// In a sequential run each stored state is restored exactly once, at the ask after the tell that stored it; + /// every other ask of the generation reuses that object. Optuna restores on every ask (199 times here). + /// + [TestMethod] + public void ASequentialRun_RestoresEachStoredStateOnce_AtTheAskAfterItsTell() + { + var sampler = new CmaEsSampler(seed: 3, popsize: 8); + var study = Optuna.CreateStudy(sampler: sampler); + study.Optimize(Sphere(5, 0.7), nTrials: 200); + + int tellers = study.Trials.Count(HoldsState); + tellers.Should().BeGreaterThan(20); + int expected = HoldsState(study.Trials[^1]) ? tellers - 1 : tellers; + sampler.RestoreCount.Should().Be(expected); + } + + /// + /// A trial that told a generation and then failed never becomes the latest state holder, so the next ask + /// restores the previous state again (the cache was dropped at the tell) and tells the same solutions: the state + /// it stores equals the failed teller's, byte for byte. The restore count keeps its one-per-tell contract. + /// + /// + /// Thrown by the objective on purpose for every other trial that told a generation, and caught by + /// Optimize, so it never leaves the test. + /// + [TestMethod] + public void AFailedTeller_ItsGenerationIsToldAgainFromThePreviousState_IdenticalState() + { + var sampler = new CmaEsSampler(seed: 5, popsize: 6); + var study = Optuna.CreateStudy(sampler: sampler); + var sphere = Sphere(4, -1.2); + int tellersSeen = 0; + study.Optimize(t => + { + double v = sphere(t); + // The trial told a generation if its sampling stored a state; every other such trial fails afterwards, so + // the failed-teller path is exercised for sure (a fixed failure pattern can miss every teller). + bool told = study.Storage.GetTrialSystemAttrs(t.TrialId).ContainsKey("cma:optimizer:0"); + if (told && tellersSeen++ % 2 == 0) + throw new InvalidOperationException("planned failure"); + return v; + }, nTrials: 300, @catch: new[] { typeof(InvalidOperationException) }); + + var trials = study.Trials; + int failedTellers = 0; + for (int i = 0; i + 1 < trials.Count; i++) + { + if (trials[i].State != TrialState.Fail || !HoldsState(trials[i])) + continue; + failedTellers++; + HoldsState(trials[i + 1]).Should().BeTrue($"trial {i + 1} tells the generation trial {i} told before failing"); + StoredState(trials[i + 1]).Should().Be(StoredState(trials[i]), + $"the same previous state and the same solutions give the same state (trials {i} and {i + 1})"); + } + failedTellers.Should().BeGreaterThan(0, "the scenario must exercise a failed teller"); + + // One restore follows each tell, at the next ask, with one exception: when no completed trial holds a state yet + // (the very first teller failed), the next ask starts a fresh optimizer instead of restoring one. + int expected = 0; + bool holderExists = false; + for (int i = 0; i + 1 < trials.Count; i++) + { + if (!HoldsState(trials[i])) + continue; + if (trials[i].State == TrialState.Complete) + holderExists = true; + if (holderExists) + expected++; + } + sampler.RestoreCount.Should().Be(expected); + } + + /// + /// One sampler used for a second study on another in-memory storage. Trial ids restart at 0 there, so the second + /// study's state holders reuse the first study's ids; the history starts over on the new storage and drops the + /// cached optimizer, so every state is restored from the second study's own storage. A stale cache would answer + /// with the first study's optimizer once the ids collide, and the count would come out one short. + /// + [TestMethod] + public void ASecondStudyOnAnotherStorage_NeverReusesTheFirstStudysOptimizer() + { + var sampler = new CmaEsSampler(seed: 1, popsize: 6); + var first = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: sampler); + first.Optimize(Sphere(3, 1.0), nTrials: 30); + int afterFirst = sampler.RestoreCount; + + var second = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: sampler); + second.Optimize(Sphere(3, -2.0), nTrials: 80); + + int tellers = second.Trials.Count(HoldsState); + int expected = HoldsState(second.Trials[^1]) ? tellers - 1 : tellers; + (sampler.RestoreCount - afterFirst).Should().Be(expected); + second.Trials.Skip(1).Should().OnlyContain(t => t.SystemAttrs.ContainsKey("cma:generation"), + "every trial after the startup trial is sampled by CMA-ES from the second study's own state"); + } + + /// + /// Study.GetTrialsForSampler reads an in-memory study through its zero-copy view, and any other storage + /// through Optuna's read (here a journal file), so a database-backed storage gets no extra round trip. Both + /// return the same trials in the same order. + /// + [TestMethod] + public void GetTrialsForSampler_IsTheZeroCopyViewInMemory_AndOptunasReadElsewhere() + { + var inMemory = Optuna.CreateStudy(storage: new InMemoryStorage()); + inMemory.Optimize(t => t.SuggestFloat("x", 0, 1), nTrials: 5); + inMemory.Storage.HasZeroCopyTrialsView.Should().BeTrue(); + var view = inMemory.GetTrialsForSampler(useCache: false); + view.Should().BeOfType(); + view.Select(t => t.Number).Should().Equal(Enumerable.Range(0, 5)); + + string path = Path.Combine(Path.GetTempPath(), $"optunasharp-sampler-read-{Guid.NewGuid():N}.log"); + try + { + var journal = Optuna.CreateStudy(storage: new JournalStorage(new JournalFileBackend(path))); + journal.Optimize(t => t.SuggestFloat("x", 0, 1), nTrials: 5); + journal.Storage.HasZeroCopyTrialsView.Should().BeFalse(); + var read = journal.GetTrialsForSampler(useCache: false); + read.Should().NotBeOfType(); + read.Select(t => t.Number).Should().Equal(Enumerable.Range(0, 5)); + } + finally + { + File.Delete(path); + } + } + + /// + /// The default is Optuna's path: a sampler built without the opt-in produces the same trials as one built with + /// reuseToldOptimizer: false, and restores once per tell (the default-path contract above). + /// + [TestMethod] + public void ReuseToldOptimizer_IsOffByDefault() + { + var byDefault = new CmaEsSampler(seed: 21, popsize: 6); + var first = Optuna.CreateStudy(sampler: byDefault); + first.Optimize(Sphere(4, 0.4), nTrials: 120); + var second = Optuna.CreateStudy(sampler: new CmaEsSampler(seed: 21, popsize: 6, reuseToldOptimizer: false)); + second.Optimize(Sphere(4, 0.4), nTrials: 120); + + second.Trials.Select(t => (t.Params["x0"], t.Params["x3"], t.Value)) + .Should().Equal(first.Trials.Select(t => (t.Params["x0"], t.Params["x3"], t.Value))); + int tellers = first.Trials.Count(HoldsState); + byDefault.RestoreCount.Should().Be(HoldsState(first.Trials[^1]) ? tellers - 1 : tellers); + } + + /// + /// With the opt-in, the told optimizer keeps answering, so a sequential run restores a stored state only once: + /// after the first tell, whose optimizer was fresh rather than restored and so is not kept. + /// + [TestMethod] + public void ReuseToldOptimizer_RestoresOnlyOnce_InASequentialRun() + { + var sampler = new CmaEsSampler(seed: 3, popsize: 8, reuseToldOptimizer: true); + var study = Optuna.CreateStudy(sampler: sampler); + study.Optimize(Sphere(5, 0.7), nTrials: 200); + + study.Trials.Count(HoldsState).Should().BeGreaterThan(20, "the state is still stored every generation"); + sampler.RestoreCount.Should().Be(1); + } + + /// + /// With the opt-in, a failed teller never becomes the latest state holder, so its told object (cached under its + /// id) never answers: the next ask restores the previous holder's stored state, once per failed teller. + /// + /// + /// Thrown by the objective on purpose for every other trial that told a generation, and caught by + /// Optimize, so it never leaves the test. + /// + [TestMethod] + public void ReuseToldOptimizer_AFailedTeller_TheNextAskRestoresThePreviousState() + { + var sampler = new CmaEsSampler(seed: 5, popsize: 6, reuseToldOptimizer: true); + var study = Optuna.CreateStudy(sampler: sampler); + var sphere = Sphere(4, -1.2); + int tellersSeen = 0; + study.Optimize(t => + { + double v = sphere(t); + bool told = study.Storage.GetTrialSystemAttrs(t.TrialId).ContainsKey("cma:optimizer:0"); + if (told && tellersSeen++ % 2 == 0) + throw new InvalidOperationException("planned failure"); + return v; + }, nTrials: 300, @catch: new[] { typeof(InvalidOperationException) }); + + // One restore after the first completed tell (its optimizer was fresh, so it was not kept), then one after each + // later failed teller. + var trials = study.Trials; + int expected = 0; + bool holderExists = false; + for (int i = 0; i + 1 < trials.Count; i++) + { + if (!HoldsState(trials[i])) + continue; + if (trials[i].State == TrialState.Complete && !holderExists) + { + holderExists = true; + expected++; + } + else if (trials[i].State == TrialState.Fail && holderExists) + { + expected++; + } + } + expected.Should().BeGreaterThan(2, "the scenario must exercise failed tellers after the first holder"); + sampler.RestoreCount.Should().Be(expected); + } + + /// + /// With the opt-in, every generation still stores Optuna's state, so a default sampler (or a Python worker) + /// resumes the study from storage: the resumed trials continue the generations where the reusing sampler left + /// them. + /// + [TestMethod] + public void ReuseToldOptimizer_StoresOptunasState_SoADefaultSamplerResumesTheStudy() + { + var storage = new InMemoryStorage(); + var first = Optuna.CreateStudy(storage: storage, studyName: "resume", + sampler: new CmaEsSampler(seed: 8, popsize: 6, reuseToldOptimizer: true)); + first.Optimize(Sphere(3, 1.5), nTrials: 90); + long lastGeneration = first.Trials.Where(t => t.SystemAttrs.ContainsKey("cma:generation")) + .Max(t => Convert.ToInt64(t.SystemAttrs["cma:generation"])); + + var resumed = Optuna.LoadStudy("resume", storage, sampler: new CmaEsSampler(seed: 9, popsize: 6)); + resumed.Optimize(Sphere(3, 1.5), nTrials: 60); + + var added = resumed.Trials.Skip(90).ToList(); + added.Should().OnlyContain(t => t.State == TrialState.Complete && t.SystemAttrs.ContainsKey("cma:generation")); + Convert.ToInt64(added[0].SystemAttrs["cma:generation"]).Should().BeGreaterThanOrEqualTo(lastGeneration, + "the resumed sampler restores the stored state instead of starting over"); + added.Max(t => Convert.ToInt64(t.SystemAttrs["cma:generation"])).Should().BeGreaterThan(lastGeneration); + } + + /// + /// The opt-in changes the run but not its quality: on a 10-D sphere both paths reach the optimum to within 1e-6 + /// for every seed. (Measured on 40-D, 10 seeds: best-value ratio median 0.79, range 0.51-1.48.) + /// + /// Sampler seed. + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void ReuseToldOptimizer_ConvergesLikeTheDefault(int seed) + { + foreach (bool reuse in new[] { false, true }) + { + var study = Optuna.CreateStudy(sampler: new CmaEsSampler(seed: seed, reuseToldOptimizer: reuse)); + study.Optimize(Sphere(10, 0.25), nTrials: 1500); + study.BestValue.Should().BeLessThan(1e-6, $"reuseToldOptimizer: {reuse}, seed {seed}"); + } + } + + /// + /// Per-worker samplers (each with its own history and cache) and a shared sampler under eight workers both + /// finish a study and keep telling generations: the cache adds no state that crosses workers. + /// + /// Shared (one sampler behind its lock) or per-worker instances. + [TestMethod] + [DataRow(SamplerConcurrency.Shared)] + [DataRow(SamplerConcurrency.PerWorker)] + public void ParallelWorkers_CompleteAndKeepTellingGenerations(SamplerConcurrency concurrency) + { + var study = Optuna.CreateStudy(sampler: new CmaEsSampler(seed: 7, popsize: 8)); + study.Optimize(Sphere(6, 0.3), nTrials: 400, nJobs: 8, + executorOptions: new TrialExecutorOptions { SamplerConcurrency = concurrency }); + + study.Trials.Should().HaveCount(400); + study.Trials.Should().OnlyContain(t => t.State == TrialState.Complete); + study.Trials.Count(HoldsState).Should().BeGreaterThan(30, "generations keep being told under concurrency"); + } +} diff --git a/test/OptunaSharp.Tests/Samplers/CmaTrialHistoryTests.cs b/test/OptunaSharp.Tests/Samplers/CmaTrialHistoryTests.cs new file mode 100644 index 0000000..6f2c11c --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/CmaTrialHistoryTests.cs @@ -0,0 +1,289 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Collections; +using OptunaSharp.Distributions; +using OptunaSharp.Samplers.CmaEs; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// , the incremental history CmaEsSampler reads on every ask (Rustuna plan 13, +/// item S2). Every assertion compares it with a naive re-implementation of Optuna's per-ask scans +/// (_get_trials, _restore_optimizer's backward scan, _get_solution_trials) over the same +/// history, so "same trials, same order" is checked at every step of randomly evolving studies: trials created, +/// finished out of order, failed, pruned with and without intermediate values, stuck running. +/// +[TestClass] +public class CmaTrialHistoryTests +{ + private const string GenerationKey = "cma:generation"; + private const string OptimizerPrefix = "cma:optimizer"; + + /// The sampler's completed-trial filter, as CmaEsSampler.AsSamplerTrial applies it. + /// Whether pruned trials with reported values count. + /// The filter: the trial, a value-rewritten copy of a pruned trial, or null. + private static Func Filter(bool considerPruned) => t => + { + if (t.State == TrialState.Complete) + return t; + if (t.State == TrialState.Pruned && t.IntermediateValues.Count > 0 && considerPruned) + { + var copy = t.DeepCopy(); + copy.Value = t.IntermediateValues[t.IntermediateValues.Keys.Max()]; + return copy; + } + return null; + }; + + /// A trial snapshot with the given state, optional generation stamp and optional optimizer state. + /// Trial number (also its position). + /// Storage id. + /// Trial state. + /// Generation stamp, or null for none. + /// Whether to add an optimizer-state chunk. + /// Intermediate values (for pruned trials), or null. + /// A new snapshot. + private static FrozenTrial Make(int number, int trialId, TrialState state, long? generation, bool holdsState, + OrderedDict? intermediate = null) + { + var systemAttrs = new OrderedDict(); + if (generation is not null) + systemAttrs[GenerationKey] = generation.Value; + if (holdsState) + systemAttrs[OptimizerPrefix + ":0"] = $"state-of-{number}"; + double? value = state == TrialState.Complete ? number * 0.5 : null; + return new FrozenTrial(number, state, value, DateTime.Now, state.IsFinished() ? DateTime.Now : null, + new OrderedDict { ["x"] = (double)number }, + new OrderedDict { ["x"] = new FloatDistribution(-1000, 1000) }, + new OrderedDict(), systemAttrs, + intermediate ?? new OrderedDict(), trialId); + } + + /// The naive scans, from scratch, over the current history. + /// The history. + /// The completed-trial filter. + /// The completed trials in order. + private static List NaiveCompleted(IReadOnlyList history, Func filter) + => history.Select(filter).Where(t => t is not null).Select(t => t!).ToList(); + + /// Asserts the history's three outputs equal the naive scans' (ids, values and order). + /// The incremental history after a refresh. + /// The history list it was refreshed with. + /// The completed-trial filter. + /// Step number, for the failure message. + private static void AssertMatchesNaive(CmaTrialHistory history, IReadOnlyList list, + Func filter, int step) + { + var completed = NaiveCompleted(list, filter); + history.CompletedCount.Should().Be(completed.Count, $"step {step}: the completed count"); + + FrozenTrial? naiveHolder = completed.LastOrDefault(t => t.SystemAttrs.Keys.Any(k => k.StartsWith(OptimizerPrefix, StringComparison.Ordinal))); + (history.LatestStateHolder?.TrialId).Should().Be(naiveHolder?.TrialId, $"step {step}: the latest state holder"); + + var generations = completed.Select(t => t.SystemAttrs.TryGetValue(GenerationKey, out var g) ? Convert.ToInt64(g) : -1) + .Distinct().ToList(); + generations.Add(999); // a generation nobody has + foreach (long g in generations) + { + if (g < int.MinValue || g > int.MaxValue) + continue; + var naive = completed.Where(t => (t.SystemAttrs.TryGetValue(GenerationKey, out var s) ? Convert.ToInt64(s) : -1) == g).ToList(); + var actual = history.SolutionTrials((int)g); + actual.Select(t => (t.TrialId, t.Value)).Should().Equal(naive.Select(t => (t.TrialId, t.Value)), + $"step {step}: generation {g}'s trials, in trial order, with pruned values rewritten"); + } + } + + /// + /// 600 random steps of a study: new trials, attributes written while running, trials finished out of order as + /// complete, failed or pruned. After each step the incremental history must equal the naive scans exactly. + /// + /// Seed of the step generator. + /// Whether pruned trials with reported values count, as with considerPrunedTrials. + [TestMethod] + [DataRow(1, false)] + [DataRow(2, true)] + [DataRow(3, false)] + [DataRow(4, true)] + [DataRow(5, true)] + public void RandomlyEvolvingStudies_MatchOptunasPerAskScans_AtEveryStep(int seed, bool considerPruned) + { + var rng = new Random(seed); + var filter = Filter(considerPruned); + var history = new CmaTrialHistory(filter, GenerationKey, OptimizerPrefix); + var storage = new InMemoryStorage(); + var list = new List(); + var unfinished = new List(); + + for (int step = 0; step < 600; step++) + { + double roll = rng.NextDouble(); + if (roll < 0.45 || unfinished.Count == 0) + { + // A new trial: running (or occasionally waiting, as an enqueued trial). + int number = list.Count; + var state = rng.NextDouble() < 0.1 ? TrialState.Waiting : TrialState.Running; + list.Add(Make(number, 1000 + number, state, generation: rng.Next(0, 4), holdsState: false)); + unfinished.Add(number); + } + else if (roll < 0.55) + { + // A running trial gains attributes while running (in the tail, so re-read each refresh). + int number = unfinished[rng.Next(unfinished.Count)]; + list[number] = Make(number, 1000 + number, list[number].State, generation: rng.Next(0, 4), holdsState: rng.NextDouble() < 0.3); + } + else + { + // Finish an unfinished trial, not necessarily the oldest: out-of-order completion. + int index = rng.Next(unfinished.Count); + int number = unfinished[index]; + unfinished.RemoveAt(index); + double kind = rng.NextDouble(); + var state = kind < 0.6 ? TrialState.Complete : kind < 0.8 ? TrialState.Fail : TrialState.Pruned; + OrderedDict? intermediate = null; + if (state == TrialState.Pruned && rng.NextDouble() < 0.7) + intermediate = new OrderedDict { [1] = rng.NextDouble(), [rng.Next(2, 9)] = rng.NextDouble() }; + // Copy-on-write, as the storage does: a new snapshot replaces the old one. + list[number] = Make(number, 1000 + number, state, generation: rng.Next(0, 4), holdsState: rng.NextDouble() < 0.25, intermediate); + } + + history.Refresh(list, storage, studyId: 0); + AssertMatchesNaive(history, list, filter, step); + } + } + + /// + /// A trial left running by a dead worker stops the fold: everything after it stays in the per-refresh tail, with + /// correct results; once it is failed (as fail_stale_trials does), the fold takes everything. + /// + [TestMethod] + public void AStuckRunningTrial_KeepsLaterTrialsInTheTail_AndFoldsThemOnceItFinishes() + { + var filter = Filter(considerPruned: false); + var history = new CmaTrialHistory(filter, GenerationKey, OptimizerPrefix); + var storage = new InMemoryStorage(); + var list = new List { Make(0, 0, TrialState.Running, generation: 0, holdsState: false) }; + for (int n = 1; n <= 40; n++) + list.Add(Make(n, n, TrialState.Complete, generation: n / 10, holdsState: n % 10 == 0)); + + history.Refresh(list, storage, 0); + AssertMatchesNaive(history, list, filter, step: 0); + + // The dead worker's trial is failed (as fail_stale_trials does); the fold then takes everything. + list[0] = Make(0, 0, TrialState.Fail, generation: 0, holdsState: false); + history.Refresh(list, storage, 0); + AssertMatchesNaive(history, list, filter, step: 1); + history.WasReset.Should().BeFalse(); + } + + /// + /// With considerPrunedTrials, a pruned trial's value-rewritten copy is made once, when the trial is folded, + /// and the same copy is returned afterwards; Optuna made a new one on every ask. + /// + [TestMethod] + public void APrunedTrialsCopy_IsMadeOnceWhenFolded_AndReusedByLaterRefreshes() + { + int copies = 0; + Func counting = t => + { + if (t.State == TrialState.Pruned) + copies++; + return Filter(considerPruned: true)(t); + }; + var history = new CmaTrialHistory(counting, GenerationKey, OptimizerPrefix); + var storage = new InMemoryStorage(); + var list = new List + { + Make(0, 0, TrialState.Pruned, generation: 2, holdsState: false, new OrderedDict { [3] = 7.5, [9] = 1.25 }), + }; + + history.Refresh(list, storage, 0); + var first = history.SolutionTrials(2).Single(); + first.Value.Should().Be(1.25, "the value at the largest step, as Optuna rewrites it"); + for (int i = 0; i < 5; i++) + history.Refresh(list, storage, 0); + + copies.Should().Be(1, "a folded trial is classified once"); + history.SolutionTrials(2).Single().Should().BeSameAs(first); + } + + /// + /// The fold starts over, and says so, whenever the history it describes is gone: first use, another storage + /// object, another study id, other trials at the folded positions, or a history shorter than the fold. The + /// sampler drops its optimizer cache on that signal, because trial ids from another storage can repeat. + /// + [TestMethod] + public void WasReset_IsTrueOnTheFirstRefresh_AnotherStorageOrStudy_OrAnotherHistory() + { + var filter = Filter(considerPruned: false); + var history = new CmaTrialHistory(filter, GenerationKey, OptimizerPrefix); + var storageA = new InMemoryStorage(); + var list = Enumerable.Range(0, 5).Select(n => Make(n, n, TrialState.Complete, 0, n == 4)).ToList(); + + history.Refresh(list, storageA, 0); + history.WasReset.Should().BeTrue("the first refresh folds from scratch"); + history.Refresh(list, storageA, 0); + history.WasReset.Should().BeFalse("the same history continues"); + + history.Refresh(list, new InMemoryStorage(), 0); + history.WasReset.Should().BeTrue("trial ids of another storage mean nothing here"); + history.Refresh(list, storageA, 0); + history.WasReset.Should().BeTrue("back to the first storage: the fold was replaced, so it starts over"); + history.Refresh(list, storageA, 7); + history.WasReset.Should().BeTrue("another study"); + + // Same storage and study id, but the folded positions now hold other trials (a new study under the id). + var other = Enumerable.Range(0, 5).Select(n => Make(n, 500 + n, TrialState.Complete, 1, false)).ToList(); + history.Refresh(other, storageA, 7); + history.WasReset.Should().BeTrue("the last folded trial's id changed"); + AssertMatchesNaive(history, other, filter, step: 0); + + // A shorter history than the fold (the study was replaced by a smaller one) also starts over. + history.Refresh(other.Take(2).ToList(), storageA, 7); + history.WasReset.Should().BeTrue(); + AssertMatchesNaive(history, other.Take(2).ToList(), filter, step: 1); + } + + /// + /// A state holder that finished after a still-running trial sits in the tail, and is the latest holder: Optuna's + /// backward scan over the completed trials finds it first. + /// + [TestMethod] + public void AHolderInTheTail_IsLaterThanEveryFoldedHolder() + { + var filter = Filter(considerPruned: false); + var history = new CmaTrialHistory(filter, GenerationKey, OptimizerPrefix); + var storage = new InMemoryStorage(); + var list = new List + { + Make(0, 0, TrialState.Complete, 0, holdsState: true), + Make(1, 1, TrialState.Running, 1, holdsState: false), + Make(2, 2, TrialState.Complete, 1, holdsState: true), + }; + + history.Refresh(list, storage, 0); + history.LatestStateHolder!.TrialId.Should().Be(2, "trial 2 finished after the stuck trial 1 and holds a state"); + + list[2] = Make(2, 2, TrialState.Complete, 1, holdsState: true); + list[1] = Make(1, 1, TrialState.Complete, 1, holdsState: false); + history.Refresh(list, storage, 0); + history.LatestStateHolder!.TrialId.Should().Be(2); + AssertMatchesNaive(history, list, filter, step: 1); + } + + /// The constructor and reject null arguments up front. + [TestMethod] + public void NullArguments_AreRejected() + { + Action noFilter = () => _ = new CmaTrialHistory(null!, GenerationKey, OptimizerPrefix); + noFilter.Should().Throw(); + var history = new CmaTrialHistory(Filter(false), GenerationKey, OptimizerPrefix); + Action noHistory = () => history.Refresh(null!, new InMemoryStorage(), 0); + noHistory.Should().Throw(); + Action noStorage = () => history.Refresh(new List(), null!, 0); + noStorage.Should().Throw(); + } +} From f8df1a5d5a3d7f79720f0370a46216656ce4f6c6 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Sun, 8 Nov 2026 23:45:42 +0200 Subject: [PATCH 010/145] perf(storage): write a running trial in place while no reader holds it, share identical distributions and names; same results as Optuna Plan 13 (docs/plans/13-rustuna-derived-performance.md), phase 2: items S4 and S7. S4. Optuna's _InMemoryStorage copies a trial's record and the dictionary it writes on every write (set_trial_param, set_trial_user_attr, set_trial_system_attr, set_trial_intermediate_value), so a FrozenTrial a caller holds never changes. The port did the same, which made a trial's suggestions O(d^2): the profile of an NSGA-II study with 40 parameters spent ~66 % in SetTrialParam's dictionary clones (plan 13 section 2.2). Rustuna inserts in place and clones only for readers. Now: - FrozenTrial.StorageExclusive (internal): true while the storage alone holds a record. CreateNewTrial sets it (both branches build dictionaries nobody else holds: a new running trial or a deep copy of the template). - InMemoryStorage.WritableTrialUnlocked: a write changes the stored record in place while it is exclusive; otherwise it first replaces it with a copy that owns all five dictionaries (any later write may touch any of them), so the reader's record keeps its contents. - HandOut: every read that returns the stored record itself clears the flag under the lock: GetTrial, GetBestTrial, GetAllTrials(deepcopy: false) (unfinished records only; finished ones are never written again), and through them the attribute reads and GetNTrials. Deep copies and GetTrialParam hand nothing out. - GetTrialsPrefixView passes TrialsPrefixView a read for unfinished records (new internal constructor): taken under the storage lock with HandOut. Finished records stay lock-free, as before: a state change still publishes a new record whole (SetTrialStateValues keeps Optuna's copy), so a record that reads as finished is final. - InMemoryStorage.WriteCopyCount (internal diagnostic, like CmaEsSampler.RestoreCount): writes that had to copy. In an ask/suggest/tell loop the trial's own read at Ask is the only hand-out between its suggestions: about one copy per trial instead of one per write. S7. SetTrialParam stores one distribution instance per parameter across the study's trials when the new one is identical to the bit (Distribution.IsBitwiseIdentical: the same runtime type, bounds/step compared by bits, log flag), and one string instance per parameter name (StudyInfo.ParamNames), so names built per call ($"x{i}") are not kept once per trial. Never shared: categoricals (equal choices may be different objects, 1 and 1.0 under Python equality, and a trial must report its own), a -0.0 bound (equal to 0.0, printed differently), a deprecated subclass against its base. Compatibility is still checked first, with Optuna's messages. Distributions are never mutated after construction (only the constructor sets FloatDistribution.Step), so sharing is unobservable. Docs: TrialsPrefixView remarks (liveness, the in-memory hand-out), THREADING.md section 6.1 (the snapshot guarantee as it is now kept) and the lock inventory (the view takes the in-memory storage monitor per unfinished record; it is still the last lock a sampler takes, so the order of section 5.2 holds). Tests: - InMemoryStorageInPlaceWritesTests (13 cases): every read path (GetTrial, GetAllTrials with no filter, RUNNING, WAITING, the view's indexer and enumerator, the three attribute reads, GetBestTrial of another trial) keeps its snapshot through two writes of each kind, and every write lands; deep copies leave the record writable in place; writes nobody reads between go in place (WriteCopyCount 0 over 20 parameters and 3 attributes) and each hand-out, GetNTrials included, costs exactly one copy; a state change makes the next write copy once and a templated trial never writes its template; the view returns the current record on every read and finished records as stored; a finished trial refuses writes; an ask/suggest/tell loop copies at most once per trial. S7: identical float, stepped float and int distributions shared, compatible-but-different ones keep the caller's instance; -0.0 keeps its instance and repr; categoricals never shared (a [1.0, 2.0] trial after a [1, 2] one reports 1.0); names stored once per study; IsBitwiseIdentical case by case; incompatible distributions still refused with Optuna's messages. - StorageConcurrencyCoverageTests: new ConcurrencyGun case, the view and GetTrial read while 7 shooters write their own trials in place (30 rounds): no fingerprint changes under the reader, and the writers did copy held records. The existing GetAllTrials(deepcopy: false) case's summary now describes how the in-memory storage keeps it. - Mutation check of the new tests (each reverted): the view handing out without marking fails 4 tests (including the view race case); a write copy that shares IntermediateValues fails 10; GetAllTrials(deepcopy: false) not marking fails 5 (including the existing race case). Results: the CI unit filter (OpenBugs/HighMemory/RequiresPython/Slow excluded) 2022/2022 on net8.0 and on net10.0; live-Python interop on net10.0 with the engine required 682 passed, 0 failed (24 not run here for services and Python packages this machine lacks: Redis, and the pycma bridge and OptunaHub constraint fixtures); documentation gate 0 issues. Seeded 40-parameter studies hash identically to de495f8 (every trial's parameters and values, by bits): random 2,000, NSGA-II 2,000, CMA-ES 1,500, TPE 150 and MOTPE 120 trials. --- docs/THREADING.md | 12 +- .../Distributions/Distribution.cs | 40 ++ src/OptunaSharp.Core/FrozenTrial.cs | 20 + .../Storages/InMemoryStorage.cs | 140 ++++- .../Storages/TrialsPrefixView.cs | 58 +- .../StorageConcurrencyCoverageTests.cs | 78 ++- .../InMemoryStorageInPlaceWritesTests.cs | 502 ++++++++++++++++++ 7 files changed, 811 insertions(+), 39 deletions(-) create mode 100644 test/OptunaSharp.Tests/Storages/InMemoryStorageInPlaceWritesTests.cs diff --git a/docs/THREADING.md b/docs/THREADING.md index 84937bb..856ef53 100644 --- a/docs/THREADING.md +++ b/docs/THREADING.md @@ -460,7 +460,7 @@ trial, and the others skip it. | Pruner monitor | `BasePruner.SyncRoot`, one per pruner instance | `Prune` and `FilterStudy` | yes | | Sampler-private locks | `BaseGASampler`, `JanusSampler`, `JanusDeSampler`, `MoeaDSampler`, `SmsEmoaSampler`, `RandomEmbeddingSampler`, `OptunaHubSampler` | per-study caches | taken inside the sampler monitor; `OptunaHubSampler`'s inside the GIL too | | The GIL | CPython | Python objects | taken by the bridges inside the sampler or pruner monitor; released by a thread that holds it while it waits for any lock above (§8) | -| Storage monitors | `InMemoryStorage`, `JournalStorage`, `CachedStorage`, the SQLite `RDBStorage`, the gRPC client's trial cache | the storage's in-memory state | all but `InMemoryStorage`'s: the journal's spans every round trip of an operation, SQLite's every database call, and `CachedStorage`'s and the gRPC cache's the read of new and unfinished trials | +| Storage monitors | `InMemoryStorage`, `JournalStorage`, `CachedStorage`, the SQLite `RDBStorage`, the gRPC client's trial cache | the storage's in-memory state; `InMemoryStorage`'s is also taken by its `GetTrialsPrefixView` for each unfinished record read through it (§6.1) | all but `InMemoryStorage`'s: the journal's spans every round trip of an operation, SQLite's every database call, and `CachedStorage`'s and the gRPC cache's the read of new and unfinished trials | | Journal file locks | `JournalFileBackend`'s offset-cache lock (short), and the lock file (`JournalFileSymlinkLock` or `JournalFileOpenLock`) around each append | the offset cache; one append at a time across processes | the lock file spans the write and its `fsync` | | Database locks | EF Core: `SELECT … FOR UPDATE` row locks and a PostgreSQL advisory lock for schema creation; SQLite's database lock | arbitration between storage instances and processes | yes | | Progress bar | `ProgressBar._updateLock` | the count and the line | reads the best trial | @@ -568,9 +568,13 @@ and ends, around one native call to OpenBLAS). or an `UpdateFinishedTrialError`, and tries the next trial. - `EnqueueTrial(skipIfExists: true)` checks the existing trials and then enqueues, in two steps, as in Python. Two threads enqueueing the same parameters at the same moment can therefore both enqueue them. -- `InMemoryStorage` replaces a trial's record on every update instead of changing it (copy-on-write), so - a record read with `deepcopy: false` never changes under its reader. Records read that way are the - storage's own, shared by every thread: never mutate them (§10). +- A record read with `deepcopy: false` never changes under its reader. `InMemoryStorage` writes a running + trial's record in place only while no reader holds it (plan 13, item S4): every read that returns the + stored record (`GetTrial`, `GetAllTrials(deepcopy: false)`, the attribute reads, an unfinished record + read through `GetTrialsPrefixView`) marks it under the storage lock, and the next write copies it + first, as Optuna's copy-on-write does on every write. A finished record is never written again, so the + view reads it without the lock. Records read that way are the storage's own, shared by every thread: + never mutate them (§10). ### 6.2 Trial diff --git a/src/OptunaSharp.Core/Distributions/Distribution.cs b/src/OptunaSharp.Core/Distributions/Distribution.cs index 7c465c6..c71f9e9 100644 --- a/src/OptunaSharp.Core/Distributions/Distribution.cs +++ b/src/OptunaSharp.Core/Distributions/Distribution.cs @@ -161,6 +161,46 @@ public static void CheckDistributionCompatibility(BaseDistribution distOld, Base throw new ValueError(nameof(CategoricalDistribution) + " does not support dynamic value space."); } + /// + /// Whether two distributions are interchangeable to the bit: the same runtime type and every field identical, + /// doubles compared by their bits. Stricter than on purpose, so a + /// storage may keep one shared instance for both without any observable difference (Rustuna plan 13, item S7). + /// + /// One distribution. + /// The other. + /// + /// True for the same instance, or for float or integer distributions of the same type whose bounds, step and log + /// flag match exactly. False otherwise, including every categorical pair: equal categoricals may hold different + /// choice objects (Python equality makes 1 and 1.0 equal), and swapping one for the other would change the + /// values a trial reports. + /// + /// + /// Why bitwise: Equals treats -0.0 and 0.0 as one bound, and the repr of a distribution prints them + /// differently. The runtime-type check keeps the deprecated UniformDistribution family, which derives from + /// , from being swapped for a plain one. + /// + internal static bool IsBitwiseIdentical(BaseDistribution a, BaseDistribution b) + { + if (ReferenceEquals(a, b)) + return true; + if (a.GetType() != b.GetType()) + return false; + return (a, b) switch + { + (FloatDistribution fa, FloatDistribution fb) => + BitConverter.DoubleToInt64Bits(fa.Low) == BitConverter.DoubleToInt64Bits(fb.Low) + && BitConverter.DoubleToInt64Bits(fa.High) == BitConverter.DoubleToInt64Bits(fb.High) + && fa.Log == fb.Log + && (fa.Step is null + ? fb.Step is null + : fb.Step is not null + && BitConverter.DoubleToInt64Bits(fa.Step.Value) == BitConverter.DoubleToInt64Bits(fb.Step.Value)), + (IntDistribution ia, IntDistribution ib) => + ia.Low == ib.Low && ia.High == ib.High && ia.Step == ib.Step && ia.Log == ib.Log, + _ => false, + }; + } + /// Python _get_single_value(distribution). /// Distribution defining the parameter representation and allowed values. /// The numeric lower endpoint or the sole categorical value. diff --git a/src/OptunaSharp.Core/FrozenTrial.cs b/src/OptunaSharp.Core/FrozenTrial.cs index 939eb1f..211cd1b 100644 --- a/src/OptunaSharp.Core/FrozenTrial.cs +++ b/src/OptunaSharp.Core/FrozenTrial.cs @@ -32,6 +32,26 @@ public class FrozenTrial : BaseTrial /// Storage identifier for this trial, separate from its per-study number. internal int TrialId; + /// + /// Read and written only by , under its lock: true while that storage + /// holds the only reference to this record and the record owns every one of its dictionaries, so a write may + /// change it in place instead of copying it (Rustuna plan 13, item S4). + /// + /// + /// + /// False for every record created any other way, which is the safe default: a write then copies, as + /// Optuna's copy-on-write storage always does. The storage clears it the moment a reference leaves (any + /// read that returns this object), so a caller that holds a record never sees it change. That is the + /// invariant Optuna's copying protected, kept at a fraction of the cost: a trial's suggestions write its + /// record repeatedly while nobody reads it, which made every suggestion copy both parameter dictionaries. + /// + /// + /// Not copied by or : a shallow copy shares the + /// dictionaries, and only the storage knows when it alone holds a record. + /// + /// + internal bool StorageExclusive; + // Lazy system-attrs loading used by Trial._get_latest_trial (Python: _LazyTrialSystemAttrs). private Func>? _systemAttrsLoader; diff --git a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs index fb335cc..992f7ac 100644 --- a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs +++ b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs @@ -214,6 +214,9 @@ public override int CreateNewTrial(int studyId, FrozenTrial? templateTrial = nul _maxTrialId += 1; trial.SetNumber(_studies[studyId].Trials.Count); trial.TrialId = trialId; + // Both branches built a record with dictionaries nobody else holds, so its writes may go in place until a + // reader receives it (see FrozenTrial.StorageExclusive). + trial.StorageExclusive = true; _trialIdToStudyIdAndNumber[trialId] = (studyId, trial.Number); _studies[studyId].Trials.Add(trial); UpdateCache(trialId, studyId); @@ -250,24 +253,91 @@ public override void SetTrialParam( CheckTrialIsUpdatable(trialId, trial.State); - int studyId = _trialIdToStudyIdAndNumber[trialId].StudyId; + var study = _studies[_trialIdToStudyIdAndNumber[trialId].StudyId]; // Check param distribution compatibility with previous trial(s). - if (_studies[studyId].ParamDistribution.TryGetValue(paramName, out var existing)) + if (study.ParamDistribution.TryGetValue(paramName, out var existing)) + { Distribution.CheckDistributionCompatibility(existing, distribution); + // Share one instance per parameter across the study's trials when the new one is the same distribution + // to the bit (plan 13, item S7): memory only, the trial still holds a distribution identical in every + // field. Bitwise, so a -0.0 bound or a categorical with equal-but-different choices (1 vs 1.0, which + // Python calls equal) keeps the caller's own instance and prints as Optuna would. + if (Distribution.IsBitwiseIdentical(existing, distribution)) + distribution = existing; + } // Set param distribution. - _studies[studyId].ParamDistribution[paramName] = distribution; - - // Set param. - trial = trial.ShallowCopy(); - trial.ParamsDict = trial.ParamsDict.ShallowClone(); - trial.ParamsDict[paramName] = distribution.ToExternalRepr(paramValueInternal); - trial.DistributionsDict = trial.DistributionsDict.ShallowClone(); - trial.DistributionsDict[paramName] = distribution; - SetTrial(trialId, trial); + study.ParamDistribution[paramName] = distribution; + // One string instance per parameter name across the study (plan 13, S7): names built per call, such as + // $"x{i}", would otherwise keep a copy alive in every trial. + if (!study.ParamNames.TryGetValue(paramName, out string? name)) + study.ParamNames[paramName] = name = paramName; + + // Set param. In place while the storage alone holds the record; otherwise an own copy first, as Optuna's + // copy-on-write does (WritableTrialUnlocked). + trial = WritableTrialUnlocked(trialId); + trial.ParamsDict[name] = distribution.ToExternalRepr(paramValueInternal); + trial.DistributionsDict[name] = distribution; } } + /// + /// The record of in a form this write may change: the stored object itself while the + /// storage alone holds it, otherwise a copy that owns all of its dictionaries, stored in its place (Rustuna plan + /// 13, item S4). + /// + /// The trial being written; the caller holds the lock and has checked it is updatable. + /// A record the caller may change in place; it stays the stored record. + /// + /// + /// Optuna copies the record and the dictionary it writes on every write, so a reader holding the old record + /// never sees a change. The copy here happens only when a reader may hold the record (any read clears + /// ), and it clones all five dictionaries, because a later write + /// may touch any of them in place. A trial's suggestions, which nobody reads between, then cost one copy + /// instead of one per suggestion. + /// + /// + /// State changes () keep copying: a finished record is read without the + /// lock through , so it must be published whole, never changed in place. + /// + /// + private FrozenTrial WritableTrialUnlocked(int trialId) + { + var trial = GetTrialUnlocked(trialId); + if (trial.StorageExclusive) + return trial; + + var own = trial.ShallowCopy(); + own.ParamsDict = trial.ParamsDict.ShallowClone(); + own.DistributionsDict = trial.DistributionsDict.ShallowClone(); + own.UserAttrsDict = trial.UserAttrsDict.ShallowClone(); + own.SystemAttrsDict = trial.SystemAttrsDict.ShallowClone(); + own.IntermediateValues = trial.IntermediateValues.ShallowClone(); + own.StorageExclusive = true; + SetTrial(trialId, own); + WriteCopyCount++; + return own; + } + + /// + /// How many writes had to copy a trial's record first because a reader may hold it (Rustuna plan 13, item S4): + /// a diagnostic the tests read to prove that suggestions nobody reads between are written in place. + /// + /// Counted under the storage's lock; read it while no write is running, or the value is approximate. + internal int WriteCopyCount { get; private set; } + + /// + /// Marks a stored record as held by a reader, so the next write copies it first: every path that returns a + /// stored record (rather than a copy) goes through here. + /// + /// A record of this storage, read under the lock. + /// , for chaining at the return statement. + private static FrozenTrial HandOut(FrozenTrial trial) + { + trial.StorageExclusive = false; + return trial; + } + /// Storage identifier of the study; distinct from its name. /// Zero-based trial number within the study, not a storage identifier. /// @@ -317,7 +387,8 @@ public override FrozenTrial GetBestTrial(int studyId) throw new ValueError("No trials are completed yet."); if (_studies[studyId].Directions.Count > 1) throw new RuntimeError("Best trial can be obtained only for single-objective optimization."); - return GetTrialUnlocked(bestTrialId.Value); + // A completed record is never written again, but handing it out keeps the rule uniform. + return HandOut(GetTrialUnlocked(bestTrialId.Value)); } } @@ -430,10 +501,8 @@ public override void SetTrialIntermediateValue(int trialId, long step, double in var trial = GetTrialUnlocked(trialId); CheckTrialIsUpdatable(trialId, trial.State); - trial = trial.ShallowCopy(); - trial.IntermediateValues = trial.IntermediateValues.ShallowClone(); - trial.IntermediateValues[step] = intermediateValue; - SetTrial(trialId, trial); + // In place unless a reader may hold the record (WritableTrialUnlocked). + WritableTrialUnlocked(trialId).IntermediateValues[step] = intermediateValue; } } @@ -449,10 +518,8 @@ public override void SetTrialUserAttr(int trialId, string key, object? value) var trial = GetTrialUnlocked(trialId); CheckTrialIsUpdatable(trialId, trial.State); - trial = trial.ShallowCopy(); - trial.UserAttrsDict = trial.UserAttrsDict.ShallowClone(); - trial.UserAttrsDict[key] = value; - SetTrial(trialId, trial); + // In place unless a reader may hold the record (WritableTrialUnlocked). + WritableTrialUnlocked(trialId).UserAttrsDict[key] = value; } } @@ -467,10 +534,8 @@ public override void SetTrialSystemAttr(int trialId, string key, object? value) var trial = GetTrialUnlocked(trialId); CheckTrialIsUpdatable(trialId, trial.State); - trial = trial.ShallowCopy(); - trial.SystemAttrsDict = trial.SystemAttrsDict.ShallowClone(); - trial.SystemAttrsDict[key] = value; - SetTrial(trialId, trial); + // In place unless a reader may hold the record (WritableTrialUnlocked). + WritableTrialUnlocked(trialId).SystemAttrsDict[key] = value; } } @@ -480,7 +545,8 @@ public override FrozenTrial GetTrial(int trialId) { lock (_lock) { - return GetTrialUnlocked(trialId); + // The caller receives the stored record itself, so later writes must copy it first. + return HandOut(GetTrialUnlocked(trialId)); } } @@ -517,7 +583,13 @@ public override IReadOnlyList GetTrialsPrefixView(int studyId) { CheckStudyId(studyId); List trials = _studies[studyId].Trials; - return new TrialsPrefixView(trials, trials.Count); + // An unfinished record may be written in place while the storage alone holds it (WritableTrialUnlocked), + // so the view reads one under the lock and hands it out; finished records stay lock-free. + return new TrialsPrefixView(trials, trials.Count, index => + { + lock (_lock) + return HandOut(trials[index]); + }); } } @@ -568,7 +640,18 @@ public override List GetAllTrials( } if (deepcopy) + { trials = trials.Select(t => t.DeepCopy()).ToList(); + } + else + { + // The caller receives the stored records themselves. Finished ones are never written again; an + // unfinished one must be copied before its next write (WritableTrialUnlocked). Only those are marked, + // which keeps this pass to the few running and waiting trials' fields. + foreach (var trial in trials) + if (!trial.State.IsFinished()) + HandOut(trial); + } return trials; } @@ -607,6 +690,11 @@ internal StudyInfo(string name, List directions) internal List Trials { get; } = new(); /// Previously recorded distributions used to validate parameters across trials. internal Dictionary ParamDistribution { get; } = new(); + /// + /// The study's one string instance per parameter name, used as the key in every trial's dictionaries so + /// names built per suggestion ($"x{i}") do not keep a copy alive in each trial (plan 13, S7). + /// + internal Dictionary ParamNames { get; } = new(); /// Copy-on-write user attributes for the study. internal OrderedDict UserAttrs { get; set; } = new(); /// Copy-on-write internal attributes for the study. diff --git a/src/OptunaSharp.Core/Storages/TrialsPrefixView.cs b/src/OptunaSharp.Core/Storages/TrialsPrefixView.cs index 819b3b5..42bec80 100644 --- a/src/OptunaSharp.Core/Storages/TrialsPrefixView.cs +++ b/src/OptunaSharp.Core/Storages/TrialsPrefixView.cs @@ -14,10 +14,15 @@ namespace OptunaSharp.Storages; /// /// Liveness. The window's membership is frozen (indices 0..Count-1 at creation /// time — appends after creation are invisible), but each read returns the trial's current -/// snapshot instance: storages that copy-on-write a trial on update (as -/// does) are seen through this view at their latest state, unlike a -/// copied list whose elements go stale. Samplers want exactly that — the freshest attrs of a frozen -/// membership. +/// record: a storage that replaces a trial's record on update is seen through this view at its latest +/// state, unlike a copied list whose elements go stale. Samplers want exactly that — the freshest attrs +/// of a frozen membership. +/// +/// +/// In-memory records. writes a running trial's record in place +/// while no reader holds it (plan 13, item S4), so this view reads an unfinished record under the +/// storage's lock and marks it as held: the storage copies it before its next write, and the record the +/// caller got never changes. A finished record is never written again, so it is read without the lock. /// /// /// Footgun — concurrency. Reads are lock-free against a backing whose @@ -32,17 +37,58 @@ public sealed class TrialsPrefixView : IReadOnlyList { private readonly List _trials; + /// + /// The storage's own read for a trial that is not finished, or null to read every trial straight from the list. + /// passes one that takes its lock and marks the record as held by a reader, so the + /// storage copies it before its next write (Rustuna plan 13, item S4: a running record nobody holds is written + /// in place). Finished records are never written again, so they stay lock-free. + /// + private readonly Func? _readUnfinished; + /// Wraps the first elements of . /// The live storage list; borrowed, never mutated through this view. /// The frozen membership size; must not exceed the list's length at any read. /// is null. /// is negative. public TrialsPrefixView(List trials, int count) + : this(trials, count, readUnfinished: null) + { + } + + /// + /// Wraps the first elements of , reading unfinished trials + /// through the storage instead of straight from the list. + /// + /// The live storage list; borrowed, never mutated through this view. + /// The frozen membership size; must not exceed the list's length at any read. + /// + /// Called with the position of a trial whose record read from the list is not finished; returns the record to + /// hand out, read under the storage's lock. Null reads every position straight from the list. + /// + /// is null. + /// is negative. + internal TrialsPrefixView(List trials, int count, Func? readUnfinished) { _trials = trials ?? throw new ArgumentNullException(nameof(trials)); if (count < 0) throw new ArgumentOutOfRangeException(nameof(count), count, "A prefix length cannot be negative."); Count = count; + _readUnfinished = readUnfinished; + } + + /// Reads one position, through when the record there is not finished. + /// A position below . + /// The record to hand out. + /// + /// The lock-free first read is safe: a state change publishes a new record whole, so a record that reads as + /// finished is final; one that reads as unfinished may be changing in place, so it is read again under the lock. + /// + private FrozenTrial Read(int index) + { + FrozenTrial trial = _trials[index]; + if (_readUnfinished is not null && !trial.State.IsFinished()) + return _readUnfinished(index); + return trial; } /// The frozen membership size — appends to the backing list after construction are not visible. @@ -58,7 +104,7 @@ public FrozenTrial this[int index] { if ((uint)index >= (uint)Count) throw new ArgumentOutOfRangeException(nameof(index), index, $"The view holds {Count} trials."); - return _trials[index]; + return Read(index); } } @@ -67,7 +113,7 @@ public FrozenTrial this[int index] public IEnumerator GetEnumerator() { for (int i = 0; i < Count; i++) - yield return _trials[i]; + yield return Read(i); } /// diff --git a/test/OptunaSharp.Tests/Concurrency/StorageConcurrencyCoverageTests.cs b/test/OptunaSharp.Tests/Concurrency/StorageConcurrencyCoverageTests.cs index 42367c6..3596dbc 100644 --- a/test/OptunaSharp.Tests/Concurrency/StorageConcurrencyCoverageTests.cs +++ b/test/OptunaSharp.Tests/Concurrency/StorageConcurrencyCoverageTests.cs @@ -58,9 +58,10 @@ public void TearDown() /// /// Shooter 0 reads every trial without a deep copy, fingerprints each record, waits a moment, and fingerprints the /// same record objects again, over and over. Meanwhile every other shooter writes its own trial (attributes, - /// reports, parameters). The two fingerprints always match: the storage replaces a record on every update instead - /// of changing it, so a record a reader holds is never changed under it, as Optuna documents for - /// deepcopy=False. + /// reports, parameters). The two fingerprints always match: a record a reader holds is never changed under it, as + /// Optuna documents for deepcopy=False. The journal replaces a record on every update; the in-memory storage + /// writes a running record in place only while no reader holds it, and the read marks it, so the next write copies + /// it first (plan 13, item S4). /// /// memory or journal. [TestMethod] @@ -118,6 +119,77 @@ public void RecordReadWithoutADeepCopy_NeverChangesUnderItsReader_WhileOthersWri changedUnderReader.Should().Be(0, "a record read without a deep copy is never changed under its reader"); } + /// + /// Plan 13, item S4, through the two reads it added a hand-out to. The in-memory storage writes a running record in + /// place while no reader holds it. Shooter 0 reads the round's trials through the zero-copy view + /// () or through , fingerprints the + /// records it holds, waits a moment and fingerprints them again, over and over, while every other shooter writes + /// its own trial. The two fingerprints always match, and the writers did copy records the reader held, so the race + /// the hand-out guards against was in play. + /// + /// view or get-trial. + [TestMethod] + [DataRow("view")] + [DataRow("get-trial")] + public void RecordReadThroughTheViewOrGetTrial_NeverChangesUnderItsReader_WhileOthersWriteInPlace(string readPath) + { + var storage = new InMemoryStorage(); + Study study = Optuna.CreateStudy(storage: storage, sampler: new RandomSampler(22)); + const int rounds = 30, writes = 30; + var trials = new Trial[Threads]; + int changedUnderReader = 0, reads = 0, firstNumberOfRound = 0; + + using var quiet = new QuietLogs(); + using var gun = new ConcurrencyGun(Threads); + GunReport report = gun.Fire(rounds, + shot => + { + if (shot.Thread == 0) + { + for (int pass = 0; pass < writes * 4; pass++) + { + // Only this round's trials: older ones are finished and no longer written. + List records = readPath == "view" + ? storage.GetTrialsPrefixView(study.StudyId).Skip(firstNumberOfRound).ToList() + : trials.Skip(1).Select(t => storage.GetTrial(t.TrialId)).ToList(); + string[] before = records.Select(Fingerprint).ToArray(); + Thread.SpinWait(2_000); + string[] after = records.Select(Fingerprint).ToArray(); + if (!before.SequenceEqual(after)) + Interlocked.Increment(ref changedUnderReader); + Interlocked.Increment(ref reads); + } + return; + } + Trial trial = trials[shot.Thread]; + for (int k = 0; k < writes; k++) + { + trial.SetUserAttr($"a{k}", k); + trial.Report(k, step: k); + trial.SuggestFloat($"p{k}", 0, 1); + } + }, + beforeRound: _ => + { + for (int t = 1; t < Threads; t++) + trials[t] = study.Ask(); + firstNumberOfRound = trials[1].Number; + }, + afterRound: _ => + { + // Each round's trials finish, so the next round's reads look only at fresh running records. + for (int t = 1; t < Threads; t++) + study.Tell(trials[t], 0.0); + return false; + }); + + report.AssertClean(); + reads.Should().BeGreaterThan(0); + changedUnderReader.Should().Be(0, $"a record read through {readPath} is never changed under its reader"); + storage.WriteCopyCount.Should().BeGreaterThan(rounds * (Threads - 1), + "beyond each trial's own read at Ask, the reader's hand-outs made the writers copy records it held"); + } + /// /// Every shooter writes a trial of its own at once (user and system attributes, reports, parameters) and then tells /// it. Every write lands, and for a journal a storage that replays the file from scratch sees exactly what the live diff --git a/test/OptunaSharp.Tests/Storages/InMemoryStorageInPlaceWritesTests.cs b/test/OptunaSharp.Tests/Storages/InMemoryStorageInPlaceWritesTests.cs new file mode 100644 index 0000000..ac481e9 --- /dev/null +++ b/test/OptunaSharp.Tests/Storages/InMemoryStorageInPlaceWritesTests.cs @@ -0,0 +1,502 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using NumSharp; +using OptunaSharp.Collections; +using OptunaSharp.Distributions; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Storages; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), phase 2. Item S4: writes a +/// running trial's record in place while no reader holds it, and copies it first otherwise, so a record any read +/// returned never changes under its reader (Optuna's copy-on-write contract). Item S7: bitwise-identical float and +/// integer distributions and the parameter names are shared across a study's trials, never a categorical, a -0.0 +/// bound or a distribution of another runtime type. +/// +[TestClass] +public class InMemoryStorageInPlaceWritesTests +{ + private static readonly StudyDirection[] Minimize = { StudyDirection.Minimize }; + + // ------------------------------------------------------------------ S4: snapshots never change + + /// + /// Every way a caller can receive a stored record (or one of its dictionaries) is read, and then each kind of write + /// lands twice in a row: a parameter, an intermediate value, a user attribute, a system attribute. What the reader + /// holds stays exactly as it was read through every write, while a fresh read sees every write. + /// + /// The read under test; see for the list. + [TestMethod] + [DataRow("get-trial")] + [DataRow("all-trials")] + [DataRow("all-trials-running")] + [DataRow("all-trials-waiting")] + [DataRow("view-indexer")] + [DataRow("view-enumerator")] + [DataRow("trial-params")] + [DataRow("trial-user-attrs")] + [DataRow("trial-system-attrs")] + [DataRow("best-trial-of-another")] + public void EveryReadPath_KeepsItsSnapshot_ThroughEveryKindOfWrite(string readPath) + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "snapshots"); + // A completed trial first, so the best-trial path has something to return and the view is longer than one. + int done = storage.CreateNewTrial(studyId); + storage.SetTrialParam(done, "x", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialStateValues(done, TrialState.Complete, new[] { 1.0 }); + + int trialId = readPath == "all-trials-waiting" + ? storage.CreateNewTrial(studyId, FrozenTrial.CreateTrial(state: TrialState.Waiting)) + : storage.CreateNewTrial(studyId); + storage.SetTrialParam(trialId, "x", 0.25, new FloatDistribution(0, 1)); + storage.SetTrialUserAttr(trialId, "seed", 1L); + + var held = new List<(Func Fingerprint, string Expected)>(); + var writes = new (string Name, Action Write)[] + { + ("param", k => storage.SetTrialParam(trialId, $"p{k}", 0.125 * k, new FloatDistribution(0, 10))), + ("intermediate", k => storage.SetTrialIntermediateValue(trialId, k, k * 1.5)), + ("user attr", k => storage.SetTrialUserAttr(trialId, $"u{k}", k)), + ("system attr", k => storage.SetTrialSystemAttr(trialId, $"s{k}", $"v{k}")), + }; + + int step = 0; + foreach (var (name, write) in writes) + { + Func fingerprint = Snapshot(storage, studyId, trialId, done, readPath); + held.Add((fingerprint, fingerprint())); + // Two in a row: the first copies (the reader holds the record), the second goes in place on the copy. + write(step++); + write(step++); + + foreach (var (heldFingerprint, expected) in held) + heldFingerprint().Should().Be(expected, $"a {readPath} snapshot must not see the {name} writes after it"); + } + + // Every write landed on the stored record. + FrozenTrial current = storage.GetTrial(trialId); + current.Params.Keys.Should().Contain(new[] { "x", "p0", "p1" }); + current.IntermediateValues.Keys.Should().Equal(2L, 3L); + current.UserAttrs.Keys.Should().Equal("seed", "u4", "u5"); + current.SystemAttrs.Keys.Should().Contain(new[] { "s6", "s7" }); + } + + /// + /// Reads a stored record, or one of its dictionaries, through and returns a function + /// that fingerprints what the reader holds, so a later call shows whether it changed. + /// + /// The storage under test. + /// The study holding both trials. + /// The running (or waiting) trial being written. + /// A completed trial of the same study. + /// + /// get-trial, all-trials (no state filter), all-trials-running, all-trials-waiting, + /// view-indexer, view-enumerator, trial-params, trial-user-attrs, + /// trial-system-attrs, or best-trial-of-another (the completed trial, held while the other is + /// written, together with the running record read through GetTrial). + /// + /// A function returning the fingerprint of what was read. + /// is not one of the listed reads. + private static Func Snapshot(InMemoryStorage storage, int studyId, int trialId, int doneId, string readPath) + { + switch (readPath) + { + case "get-trial": + return FingerprintOf(storage.GetTrial(trialId)); + case "all-trials": + return FingerprintOf(storage.GetAllTrials(studyId, deepcopy: false).Single(t => t.TrialId == trialId)); + case "all-trials-running": + return FingerprintOf(storage.GetAllTrials(studyId, deepcopy: false, states: new[] { TrialState.Running }).Single()); + case "all-trials-waiting": + return FingerprintOf(storage.GetAllTrials(studyId, deepcopy: false, states: new[] { TrialState.Waiting }).Single()); + case "view-indexer": + return FingerprintOf(storage.GetTrialsPrefixView(studyId)[1]); + case "view-enumerator": + return FingerprintOf(storage.GetTrialsPrefixView(studyId).Single(t => t.TrialId == trialId)); + case "trial-params": + return FingerprintOf(storage.GetTrialParams(trialId)); + case "trial-user-attrs": + return FingerprintOf(storage.GetTrialUserAttrs(trialId)); + case "trial-system-attrs": + return FingerprintOf(storage.GetTrialSystemAttrs(trialId)); + case "best-trial-of-another": + { + Func best = FingerprintOf(storage.GetBestTrial(studyId)); + storage.GetBestTrial(studyId).TrialId.Should().Be(doneId); + Func running = FingerprintOf(storage.GetTrial(trialId)); + return () => best() + "|" + running(); + } + default: + throw new ArgumentOutOfRangeException(nameof(readPath), readPath, "Unknown read path."); + } + } + + /// Fingerprints a record by its Python repr, which prints every dictionary, its state and values. + /// The record a reader holds. + /// A function returning the record's repr at the time of the call. + private static Func FingerprintOf(FrozenTrial trial) => () => trial.ToString(); + + /// Fingerprints a dictionary a reader holds, its keys and values in order. + /// The dictionary a read returned. + /// A function returning the dictionary's entries at the time of the call. + private static Func FingerprintOf(IReadOnlyDictionary dictionary) + => () => string.Join(";", dictionary.Select(kv => $"{kv.Key}={kv.Value}")); + + /// + /// A deep copy taken by the caller holds nothing of the storage's, so the stored record stays writable in place; + /// the copy itself never sees a later write. + /// + [TestMethod] + public void DeepCopiedReads_LeaveTheStoredRecordWritableInPlace_AndNeverChange() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "deep"); + int trialId = storage.CreateNewTrial(studyId); + storage.SetTrialParam(trialId, "x", 0.25, new FloatDistribution(0, 1)); + + FrozenTrial copy = storage.GetAllTrials(studyId, deepcopy: true).Single(); + string before = copy.ToString(); + storage.SetTrialParam(trialId, "y", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialUserAttr(trialId, "k", 1L); + + copy.ToString().Should().Be(before); + storage.WriteCopyCount.Should().Be(0, "a deep copy holds none of the stored record's dictionaries"); + } + + // ------------------------------------------------------------------ S4: in place when nobody holds the record + + /// + /// A running trial that nobody read is written in place however many writes land, and a read that does not hand out + /// the stored record (a deep copy, the internal-repr parameter, a state filter that skips the trial, making a view) + /// keeps it that way. Each read that hands the record out makes exactly the next write copy it, once; that includes + /// a count, which Optuna's base storage takes through GetAllTrials(deepcopy: false). + /// + [TestMethod] + public void WritesNobodyReadsBetween_GoInPlace_AndEachHandOutCostsOneCopy() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "in-place"); + int trialId = storage.CreateNewTrial(studyId); + + // A trial's suggestions and attributes, nobody reading between: Optuna copied the record for each of these. + for (int i = 0; i < 20; i++) + storage.SetTrialParam(trialId, $"x{i}", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialUserAttr(trialId, "u", 1L); + storage.SetTrialSystemAttr(trialId, "s", 1L); + storage.SetTrialIntermediateValue(trialId, 0, 0.5); + storage.WriteCopyCount.Should().Be(0); + + // Reads that do not hand the stored record out. + _ = storage.GetAllTrials(studyId, deepcopy: true); + _ = storage.GetTrialParam(trialId, "x0"); + _ = storage.GetAllTrials(studyId, deepcopy: false, states: new[] { TrialState.Complete }); + _ = storage.GetTrialsPrefixView(studyId); // a view hands a record out when it is read, not when it is made + WriteThrice(storage, trialId, "a"); + storage.WriteCopyCount.Should().Be(0); + + // Each read that hands it out: exactly one copy on the next write, then in place again. + int expected = 0; + foreach (Action read in new Action[] + { + () => storage.GetTrial(trialId), + () => storage.GetAllTrials(studyId, deepcopy: false), + () => storage.GetAllTrials(studyId, deepcopy: false, states: new[] { TrialState.Running }), + () => _ = storage.GetTrialsPrefixView(studyId)[0], + () => _ = storage.GetTrialsPrefixView(studyId).ToList(), + () => storage.GetTrialParams(trialId), + () => storage.GetTrialUserAttrs(trialId), + () => storage.GetTrialSystemAttrs(trialId), + () => storage.GetNTrials(studyId), + }) + { + read(); + WriteThrice(storage, trialId, $"r{expected}"); + storage.WriteCopyCount.Should().Be(++expected); + } + } + + /// Writes a parameter, a user attribute and a system attribute named after . + /// The storage under test. + /// The running trial to write. + /// Makes the names unique per call. + private static void WriteThrice(InMemoryStorage storage, int trialId, string tag) + { + storage.SetTrialParam(trialId, $"p_{tag}", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialUserAttr(trialId, $"u_{tag}", 1L); + storage.SetTrialSystemAttr(trialId, $"s_{tag}", 1L); + } + + /// + /// A state change publishes a new record that shares the old one's dictionaries, so its first write copies; a trial + /// created from a template owns deep copies, so its writes go in place and never reach the template. + /// + [TestMethod] + public void AStateChange_MakesTheNextWriteCopy_AndATemplatedTrialNeverWritesItsTemplate() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "states"); + var template = FrozenTrial.CreateTrial(state: TrialState.Waiting); + template.UserAttrsDict["from"] = "template"; + string templateBefore = template.ToString(); + + int trialId = storage.CreateNewTrial(studyId, template); + storage.SetTrialUserAttr(trialId, "k", 1L); + storage.SetTrialParam(trialId, "x", 0.5, new FloatDistribution(0, 1)); + storage.WriteCopyCount.Should().Be(0); + template.ToString().Should().Be(templateBefore, "the stored record owns copies of the template's dictionaries"); + + FrozenTrial waiting = storage.GetAllTrials(studyId, deepcopy: false, states: new[] { TrialState.Waiting }).Single(); + storage.SetTrialStateValues(trialId, TrialState.Running).Should().BeTrue(); + storage.SetTrialUserAttr(trialId, "after", 2L); + storage.SetTrialUserAttr(trialId, "after2", 3L); + storage.WriteCopyCount.Should().Be(1, "the running record shares the waiting record's dictionaries, once"); + waiting.UserAttrs.Keys.Should().Equal("from", "k"); + waiting.State.Should().Be(TrialState.Waiting); + } + + /// + /// The view keeps its documented liveness: each read returns the trial's current record, whether the write after + /// the previous read went in place or through a copy; a finished record is returned as stored, without a copy. + /// + [TestMethod] + public void TheView_ReturnsTheCurrentRecordOnEveryRead_AndFinishedRecordsAsStored() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "view"); + int trialId = storage.CreateNewTrial(studyId); + IReadOnlyList view = storage.GetTrialsPrefixView(studyId); + + storage.SetTrialParam(trialId, "x", 0.25, new FloatDistribution(0, 1)); + view[0].Params.Keys.Should().Equal("x"); // written in place before this first read + storage.SetTrialParam(trialId, "y", 0.25, new FloatDistribution(0, 1)); + view[0].Params.Keys.Should().Equal("x", "y"); // written through a copy after the read + view.Single().Params.Keys.Should().Equal("x", "y"); + + storage.SetTrialStateValues(trialId, TrialState.Complete, new[] { 2.0 }); + FrozenTrial finished = view[0]; + finished.State.Should().Be(TrialState.Complete); + view[0].Should().BeSameAs(finished, "a finished record is never written again, so it is returned as stored"); + storage.GetTrial(trialId).Should().BeSameAs(finished); + } + + /// A finished trial still refuses every write, so its record is never changed in place. + [TestMethod] + public void AFinishedTrial_RefusesWrites_SoItsRecordNeverChanges() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "finished"); + int trialId = storage.CreateNewTrial(studyId); + storage.SetTrialStateValues(trialId, TrialState.Complete, new[] { 1.0 }); + FrozenTrial record = storage.GetTrialsPrefixView(studyId)[0]; + string before = record.ToString(); + + Action param = () => storage.SetTrialParam(trialId, "x", 0.5, new FloatDistribution(0, 1)); + Action attr = () => storage.SetTrialUserAttr(trialId, "k", 1L); + Action report = () => storage.SetTrialIntermediateValue(trialId, 0, 1.0); + param.Should().Throw(); + attr.Should().Throw(); + report.Should().Throw(); + record.ToString().Should().Be(before); + storage.WriteCopyCount.Should().Be(0); + } + + /// + /// A study driven through ask/suggest/tell costs about one copy per trial instead of one per write: the trial's + /// own read when it starts is the only hand-out between its suggestions. + /// + [TestMethod] + public void AnAskTellLoop_CopiesAboutOncePerTrial_NotOncePerWrite() + { + var storage = new InMemoryStorage(); + Study study = Optuna.CreateStudy(storage: storage, sampler: new RandomSampler(3)); + const int trials = 25, parameters = 12; + for (int n = 0; n < trials; n++) + { + Trial trial = study.Ask(); + double sum = 0; + for (int i = 0; i < parameters; i++) + sum += trial.SuggestFloat($"x{i}", -1, 1); + trial.SetUserAttr("sum", sum); + study.Tell(trial, sum * sum); + } + + // Optuna copies per write: 25 x 13. One copy per trial is the trial's own read at Ask. + storage.WriteCopyCount.Should().BeLessThanOrEqualTo(trials); + study.Trials.Should().HaveCount(trials); + study.Trials.Should().OnlyContain(t => t.Params.Count == parameters && t.UserAttrs.ContainsKey("sum")); + } + + // ------------------------------------------------------------------ S7: shared distributions and names + + /// + /// Float and integer distributions identical to the bit are shared across the study's trials, so the second + /// trial's record holds the first trial's instance; a compatible but different one keeps the caller's instance. + /// + [TestMethod] + public void BitwiseIdenticalFloatAndIntDistributions_AreShared_AndOthersKeepTheCallersInstance() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "shared"); + int first = storage.CreateNewTrial(studyId); + storage.SetTrialParam(first, "f", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialParam(first, "s", 0.5, new FloatDistribution(0, 1, step: 0.25)); + storage.SetTrialParam(first, "i", 3, new IntDistribution(1, 9)); + + int second = storage.CreateNewTrial(studyId); + storage.SetTrialParam(second, "f", 0.25, new FloatDistribution(0, 1)); + storage.SetTrialParam(second, "s", 0.25, new FloatDistribution(0, 1, step: 0.25)); + storage.SetTrialParam(second, "i", 5, new IntDistribution(1, 9)); + + FrozenTrial a = storage.GetTrial(first), b = storage.GetTrial(second); + b.Distributions["f"].Should().BeSameAs(a.Distributions["f"]); + b.Distributions["s"].Should().BeSameAs(a.Distributions["s"]); + b.Distributions["i"].Should().BeSameAs(a.Distributions["i"]); + b.Params["f"].Should().Be(0.25); + b.Params["i"].Should().Be(5L); + + // Compatible (same kind and log flag) but not identical: the trial keeps what the caller passed. + var wider = new FloatDistribution(0, 2); + var stepped = new IntDistribution(1, 9, step: 2); + int third = storage.CreateNewTrial(studyId); + storage.SetTrialParam(third, "f", 1.5, wider); + storage.SetTrialParam(third, "i", 3, stepped); + FrozenTrial c = storage.GetTrial(third); + c.Distributions["f"].Should().BeSameAs(wider); + c.Distributions["i"].Should().BeSameAs(stepped); + } + + /// + /// A -0.0 bound equals 0.0 but prints differently, so a trial given one keeps its own instance and its repr shows + /// Python's -0.0. + /// + [TestMethod] + public void ANegativeZeroBound_KeepsTheCallersInstance_AndItsRepr() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "negative-zero"); + int first = storage.CreateNewTrial(studyId); + storage.SetTrialParam(first, "x", 0.5, new FloatDistribution(0.0, 1)); + + var negativeZero = new FloatDistribution(-0.0, 1); + int second = storage.CreateNewTrial(studyId); + storage.SetTrialParam(second, "x", 0.5, negativeZero); + + FrozenTrial b = storage.GetTrial(second); + b.Distributions["x"].Should().BeSameAs(negativeZero); + b.Distributions["x"].ToString().Should().Contain("low=-0.0"); + storage.GetTrial(first).Distributions["x"].ToString().Should().Contain("low=0.0"); + } + + /// + /// Categorical distributions are never shared: equal choices may be different objects (Python calls 1 and 1.0 + /// equal), and each trial must report the choice object of the distribution it was given. + /// + [TestMethod] + public void CategoricalDistributions_AreNeverShared_SoEachTrialReportsItsOwnChoices() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "categorical"); + var ints = new CategoricalDistribution(new CategoricalChoice[] { 1, 2 }); + var sameInts = new CategoricalDistribution(new CategoricalChoice[] { 1, 2 }); + var doubles = new CategoricalDistribution(new CategoricalChoice[] { 1.0, 2.0 }); + + int first = storage.CreateNewTrial(studyId); + storage.SetTrialParam(first, "c", 0, ints); + int second = storage.CreateNewTrial(studyId); + storage.SetTrialParam(second, "c", 0, sameInts); + int third = storage.CreateNewTrial(studyId); + storage.SetTrialParam(third, "c", 0, doubles); // compatible: Python equality + + storage.GetTrial(second).Distributions["c"].Should().BeSameAs(sameInts); + storage.GetTrial(third).Distributions["c"].Should().BeSameAs(doubles); + storage.GetTrial(first).Params["c"].Should().Be(1L); + storage.GetTrial(third).Params["c"].Should().BeOfType().And.Be(1.0); + } + + /// + /// Parameter names built per suggestion are stored once per study: every trial's dictionaries key on the first + /// trial's string instance, and the names still compare and print as given. + /// + [TestMethod] + public void ParameterNamesBuiltPerSuggestion_AreStoredOncePerStudy() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "names"); + var trialIds = new List(); + for (int n = 0; n < 3; n++) + { + int trialId = storage.CreateNewTrial(studyId); + trialIds.Add(trialId); + // A new string instance per call, as $"x{i}" in an objective makes one. + storage.SetTrialParam(trialId, new string("lr".ToCharArray()), 0.5, new FloatDistribution(0, 1)); + } + + string key = storage.GetTrial(trialIds[0]).Params.Keys.Single(); + foreach (int trialId in trialIds) + { + FrozenTrial trial = storage.GetTrial(trialId); + trial.Params.Keys.Single().Should().BeSameAs(key); + trial.Distributions.Keys.Single().Should().BeSameAs(key); + trial.Params.Keys.Single().Should().Be("lr"); + } + } + + /// + /// The rule behind S7, case by case: the same instance, or the same runtime type with identical bits; never a + /// categorical pair, a deprecated subclass against its base, two kinds, or bounds that differ only in the sign of + /// zero. + /// + [TestMethod] + public void IsBitwiseIdentical_TheSameInstanceOrTheSameTypeAndBits_NeverCategoricals() + { + var categorical = new CategoricalDistribution(new CategoricalChoice[] { "a", "b" }); + Distribution.IsBitwiseIdentical(categorical, categorical).Should().BeTrue(); + Distribution.IsBitwiseIdentical(categorical, new CategoricalDistribution(new CategoricalChoice[] { "a", "b" })) + .Should().BeFalse(); + + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1), new FloatDistribution(0, 1)).Should().BeTrue(); + Distribution.IsBitwiseIdentical(new FloatDistribution(1e-3, 1, log: true), new FloatDistribution(1e-3, 1, log: true)) + .Should().BeTrue(); + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1, step: 0.1), new FloatDistribution(0, 1, step: 0.1)) + .Should().BeTrue(); + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1), new FloatDistribution(-0.0, 1)).Should().BeFalse(); + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1), new FloatDistribution(0, 1, step: 0.5)).Should().BeFalse(); + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1, step: 0.5), new FloatDistribution(0, 1, step: 0.25)) + .Should().BeFalse(); + Distribution.IsBitwiseIdentical(new FloatDistribution(0.1, 1), new FloatDistribution(0.1, 1, log: true)).Should().BeFalse(); + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1), new FloatDistribution(0, 2)).Should().BeFalse(); +#pragma warning disable CS0618 // the deprecated family is the point of this case + Distribution.IsBitwiseIdentical(new FloatDistribution(0, 1), new UniformDistribution(0, 1)).Should().BeFalse(); +#pragma warning restore CS0618 + + Distribution.IsBitwiseIdentical(new IntDistribution(1, 9), new IntDistribution(1, 9)).Should().BeTrue(); + Distribution.IsBitwiseIdentical(new IntDistribution(1, 9), new IntDistribution(1, 9, step: 2)).Should().BeFalse(); + Distribution.IsBitwiseIdentical(new IntDistribution(1, 9), new IntDistribution(1, 9, log: true)).Should().BeFalse(); + Distribution.IsBitwiseIdentical(new IntDistribution(0, 1), new FloatDistribution(0, 1)).Should().BeFalse(); + } + + /// + /// A parameter's compatibility is still checked before anything is shared: another kind or another log flag is + /// refused with Optuna's message, exactly as before. + /// + [TestMethod] + public void IncompatibleDistributions_AreStillRefused_WithOptunasMessages() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "compat"); + int first = storage.CreateNewTrial(studyId); + storage.SetTrialParam(first, "x", 0.5, new FloatDistribution(0.1, 1)); + int second = storage.CreateNewTrial(studyId); + + Action kind = () => storage.SetTrialParam(second, "x", 1, new IntDistribution(0, 1)); + Action log = () => storage.SetTrialParam(second, "x", 0.5, new FloatDistribution(0.1, 1, log: true)); + kind.Should().Throw().WithMessage("Cannot set different distribution kind to the same parameter name."); + log.Should().Throw().WithMessage("Cannot set different log configuration to the same parameter name."); + storage.GetTrial(second).Params.Should().BeEmpty(); + } +} From 4464e8a59440534af134b71f1ebc5c21e0244ed4 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 00:22:37 +0200 Subject: [PATCH 011/145] perf(tpe): fused scoring kernel, one trial split per trial, zero-copy history reads; same results as before Plan 13 (docs/plans/13-rustuna-derived-performance.md), phase 3: items S6, S5 and S3 (TPE's constant-liar read, QMC, the group-decomposed search space). Every change keeps OptunaSharp's results to the bit: seeded 40-parameter studies hash identically to de495f8 (TPE 1,000 trials, MOTPE 1,000, QMC), and the live-Python TPE suite, now with 1,000- and 300-trial long runs, stays within its erf tolerance. Measured (i9-13900K, Release, power throttling opted out, one rep each unless noted; 40 float parameters, seed 0): - TPE, 1,000 trials: 10.08 s -> 4.76 s (first rep, JIT included), 3.93 s steady; allocations 4.4 GB -> 1.6 GB, gen-2 collections 254 -> 2. - MOTPE (two objectives), 1,000 trials: 41.07 s -> 15.77 s; allocations 28.7 GB -> 3.4 GB, gen-2 713 -> 1. - MOTPE, 300 trials: 3.75 s -> 2.53 s. S6, the TPE scoring kernel (Samplers/Tpe/ProbabilityDistributions.cs, ParzenEstimator.cs, Internal/TruncNorm.cs, Internal/Erf.cs): - AddContinuousLogPdf computed log(sigma), the standardized bounds and their log Gaussian mass once per candidate for every (kernel, dimension) pair: 936k Math.Log calls per trial at 1,000 trials and 40 dimensions, more than half the run. They are now computed once per pair, in grids laid out by dimension, and AccumulateDimension adds one dimension's term to every kernel's sum on Vector lanes. Each kernel's sum still runs over the dimensions in order from +0.0, lanes hold different kernels and never mix, and .NET never contracts a multiply and an add into an FMA, so every lane computes what the scalar loop computed; the remainder takes the scalar path. LogGaussMass is elementwise and the vectorized-erf switch depends only on how many elements it gets, so laying the grid out by dimension changes no element. - LogPdf takes each kernel's log weight once instead of once per candidate, and keeps its (candidates x kernels) matrix in a rented flat buffer: at 1,000 kernels it was a 192 KB large-object allocation per call, and the sampler waited on background GCs for them. - TruncNorm.LogGaussMass and Ndtr, and Erf.ComputeArray, gained span overloads over rented scratch (the array overloads now wrap them). The central elements are gathered into spans of exactly the central count, so the erf path is chosen from the same length as before; a test pins that a span of 1,999 inside a 5,000-element buffer takes the exact path. - ParzenEstimator.StableArgsort replaces Enumerable.Range(0, n).OrderBy(i => mus[i]): Array.Sort on a copy of the keys, then an index sort inside each run of equal keys (by CompareTo: NaNs together, -0.0 with 0.0), which is the same permutation the stable LINQ sort returned. - TPESampler.GetInternalRepr fills pre-sized columns by position, without the per-trial LINQ closure, the dictionary lookup per value and the list regrowth; same values in the same order. S5, the split cache (Internal/TpeTrialSplit.cs: TpeSplitCache; TPESampler): independent TPE (the default for several objectives) split the history once per parameter, and every parameter of a trial sees the same records, so a 40-parameter trial computed the same non-domination ranks and hypervolume tie-break 40 times. The cache returns the last split again only for the very same records (compared by reference, in order) of the same study and storage with the same n_below: finished records never change, and a record changes reference on any write after a reader got it (state changes publish a new record; InMemoryStorage copies a handed-out record before writing it). Returned lists are copies. One cache per sampler instance, so a PerWorker worker has its own (TPESampler.SplitCache, internal, for tests). S3: - TPE with constant liar (Optuna 5's default) read the whole history including running trials for every parameter and then dropped the current trial; that read handed out the current trial's record, so every suggestion copied it again under item S4. On a storage with a zero-copy view it now reads through the view and skips the current trial's position without reading it (TPESampler.ReadOtherTrials): the same records in the same order, minus the current one. - QMCSampler copied the COMPLETE/PRUNED history and sorted it by number on every ask to take the first one; FirstSuggestedTrial scans the history in number order through Study.GetTrialsForSampler and stops at the first match (O(1) on an in-memory storage, Optuna's read elsewhere). - GroupDecomposedSearchSpace filters the states while scanning Study.GetTrialsForSampler instead of a copied list. Tests: - TpeExactKernelTests (unit): the continuous log-pdf against a verbatim scalar copy of the replaced code, bit for bit, for every kernel count from 1 to three vectors and more (every remainder), 1-40 dimensions, log dimensions, candidates outside the bounds; 1,000-kernel grids that take the vectorized erf; an empty interval (NaN, as before); LogGaussMass over rented scratch against the replaced allocation-per-call code for 1 to 6,000 intervals around the 2,000 switch, NaN bounds and all three cases; the span erf choosing its path by the span's length; StableArgsort equal to the LINQ permutation over 300 arrays with ties, NaNs, both zeros and infinities. - TpeSplitCacheTests (unit): the cached split equals SplitTrials for every n_below over a history with completed, pruned (with and without reports), infeasible and running trials; reuse only for the same records (one fewer, another order, a rewritten running record, another study all miss); returned lists are copies; null arguments; an independent multi-objective study splits once per trial, not per parameter, with constant liar on; constant liar still sees another worker's running trial and leaves the current one out; a PerWorker instance has its own cache. - ZeroCopyHistoryReadTests (unit): QMC's search space comes from the lowest-numbered COMPLETE or PRUNED trial with trials finished out of order and running/failed trials before it; the running-trial union before anything finished; the group-decomposed space equals Optuna's decomposition of the copied, filtered history (same groups, order and distribution objects) with and without pruned trials and the cache. - Interop, live Python, [Slow]: TPE multivariate 40-D 1,000 trials and MOTPE 40-D 300 trials stay in lock-step with Optuna within the erf tolerance; NSGA-II 40 parameters 2,000 trials is bit-identical in full state (plan 13 phase 0 gates for S4-S6). - The 8 [OpenBugs] TPE/PED-ANOVA QA cases fail with byte-identical messages before and after (NaN acquisition values and NaN prior weights included), so the edge behavior they pin is unchanged. Also: OptunaHubBridgeParityTests.BenchmarkBridge_ToyProblem_ExposesSpaceAndOptimizes now seeds its TPE sampler. It ran an unseeded study (OS-entropy seed) and asserted that the best of 40 trials is below 1.0, which fails in a few percent of runs: it failed once in this phase's interop run (best 1.018) and passed eight reruns. Seeded studies are bit-identical before and after this phase, so every OS seed maps to the same outcome in both trees and the failure rate was unchanged; the test is about the bridge, so the seed is fixed (seed 0 passes). Results: the CI unit filter (OpenBugs/HighMemory/RequiresPython/Slow excluded) 2044/2044 on net8.0 and on net10.0 with S5/S6 and TPE's constant-liar read; the QMC and group-decomposition change and its 7 tests afterwards, with the QMC, group and TPE suites: 51/51. Live-Python interop on net10.0 with the engine required, the three new long runs included: 684 passed, 24 not run here (Redis, the pycma bridge and OptunaHub constraint fixtures), 1 failed: the stochastic OptunaHub toy test above. Long runs: TPE 1,000 trials 1 min 26 s, MOTPE 300 trials 58 s, NSGA-II 2,000 trials 16.5 s, all passing. Documentation gate 0 issues (5,609 declarations). --- src/OptunaSharp.Core/Internal/Erf.cs | 17 +- .../Internal/TpeTrialSplit.cs | 88 ++++ src/OptunaSharp.Core/Internal/TruncNorm.cs | 122 ++++-- src/OptunaSharp.Core/Samplers/QMCSampler.cs | 34 +- src/OptunaSharp.Core/Samplers/TPESampler.cs | 102 ++++- .../Samplers/Tpe/ParzenEstimator.cs | 40 +- .../Samplers/Tpe/ProbabilityDistributions.cs | 360 ++++++++++------ .../SearchSpace/GroupDecomposedSearchSpace.cs | 9 +- .../NsgaiiSamplerParityTests.cs | 41 ++ .../OptunaHubBridgeParityTests.cs | 5 +- .../TpeSamplerParityTests.cs | 79 ++++ .../Samplers/TpeExactKernelTests.cs | 404 ++++++++++++++++++ .../Samplers/TpeSplitCacheTests.cs | 223 ++++++++++ .../Samplers/ZeroCopyHistoryReadTests.cs | 151 +++++++ 14 files changed, 1498 insertions(+), 177 deletions(-) create mode 100644 test/OptunaSharp.Tests/Samplers/TpeExactKernelTests.cs create mode 100644 test/OptunaSharp.Tests/Samplers/TpeSplitCacheTests.cs create mode 100644 test/OptunaSharp.Tests/Samplers/ZeroCopyHistoryReadTests.cs diff --git a/src/OptunaSharp.Core/Internal/Erf.cs b/src/OptunaSharp.Core/Internal/Erf.cs index 8694cc7..04db0e6 100644 --- a/src/OptunaSharp.Core/Internal/Erf.cs +++ b/src/OptunaSharp.Core/Internal/Erf.cs @@ -256,11 +256,25 @@ internal static double Erfc(double x) internal static double[] ComputeArray(double[] x) { var result = new double[x.Length]; + ComputeArray(x, result); + return result; + } + + /// + /// into a caller's buffer, for callers that reuse scratch memory (Rustuna + /// plan 13, item S6). The path is chosen by 's length exactly as the array overload chooses + /// it by the array's length, so a caller must pass a span of the logical length, never a larger rented buffer. + /// + /// The numerical input values; its length selects the exact or the vectorized path. + /// Receives one value per element of ; at least as long, and not overlapping it. + /// is shorter than . + internal static void ComputeArray(ReadOnlySpan x, Span result) + { if (x.Length < 2000) { for (int i = 0; i < x.Length; i++) result[i] = Compute(x[i]); - return result; + return; } for (int i = 0; i < x.Length; i++) @@ -313,7 +327,6 @@ internal static double[] ComputeArray(double[] x) } result[i] = Math.Sign(v) * outVal; } - return result; } /// diff --git a/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs b/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs index bf5399f..d681683 100644 --- a/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs +++ b/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs @@ -321,3 +321,91 @@ private static double[] GetReferencePoint(double[][] lossVals) return referencePoint; } } + +/// +/// The last below/above split a TPE sampler computed, reused while the same trial records are split again (Rustuna +/// plan 13, item S5). Independent TPE (the default for several objectives) splits the history once per parameter, and +/// every parameter of a trial sees the same records, so a 40-parameter trial computed the same split 40 times: about +/// a quarter of a multi-objective run, in the non-domination ranks and the hypervolume tie-break. +/// +/// +/// +/// Why the records themselves are the key. is a function of the +/// records it is given (their state, values, intermediate values and constraints), the study's directions and +/// n_below. A finished record never changes, and a record changes reference whenever its trial is written +/// after a reader got it: storages publish a new record on every state change, and InMemoryStorage +/// copies a handed-out record before writing it. So the same references in the same order, of the same study, +/// with the same n_below, split the same way, and a cached split is returned only then. Anything else +/// (a new trial, another study, a running trial written by another worker under constant liar, a storage that +/// builds new records per read) misses and splits afresh, which is what the sampler did every time before. +/// +/// +/// Not thread-safe: the sampler calls it under its own monitor (THREADING.md §6.3), and a +/// worker instance gets a cache of its own. +/// +/// +internal sealed class TpeSplitCache +{ + private OptunaSharp.Storages.BaseStorage? _storage; + private int _studyId; + private int _nBelow; + private FrozenTrial[] _trials = Array.Empty(); + private List _below = new(); + private List _above = new(); + + /// How many splits were computed rather than reused: a diagnostic the tests read. + internal int Computed { get; private set; } + + /// + /// for these records, reused when the previous call split the very same + /// records of the same study with the same . + /// + /// The study the records belong to; its storage and id are part of the key. + /// The records to split, in the order the sampler read them. + /// The number of trials the preferred group should hold. + /// New lists the caller may change, holding the preferred and the remaining records by trial number. + /// or is null. + /// Whatever throws on a miss; nothing is cached then. + internal (List Below, List Above) Split(Study study, IReadOnlyList trials, int nBelow) + { + ArgumentNullException.ThrowIfNull(study); + ArgumentNullException.ThrowIfNull(trials); + + if (!Matches(study, trials, nBelow)) + { + var (below, above) = TpeTrialSplit.SplitTrials(study, trials, nBelow); + // Stored only after the split succeeded, so a throwing split leaves the previous entry consistent. + var key = new FrozenTrial[trials.Count]; + for (int i = 0; i < key.Length; i++) + key[i] = trials[i]; + _storage = study.Storage; + _studyId = study.StudyId; + _nBelow = nBelow; + _trials = key; + _below = below; + _above = above; + Computed++; + } + + // Copies: the cached lists must survive a caller that changes what it was given. + return (new List(_below), new List(_above)); + } + + /// Whether the cached split was computed from exactly these records of this study with this size. + /// The study being sampled. + /// The records about to be split. + /// The requested size of the preferred group. + /// True when every key part matches, the records compared by reference and in order. + private bool Matches(Study study, IReadOnlyList trials, int nBelow) + { + if (_storage is null || !ReferenceEquals(_storage, study.Storage) || _studyId != study.StudyId + || _nBelow != nBelow || _trials.Length != trials.Count) + return false; + for (int i = 0; i < _trials.Length; i++) + { + if (!ReferenceEquals(_trials[i], trials[i])) + return false; + } + return true; + } +} diff --git a/src/OptunaSharp.Core/Internal/TruncNorm.cs b/src/OptunaSharp.Core/Internal/TruncNorm.cs index 04a3c35..f7e1f3a 100644 --- a/src/OptunaSharp.Core/Internal/TruncNorm.cs +++ b/src/OptunaSharp.Core/Internal/TruncNorm.cs @@ -55,16 +55,43 @@ private static double NdtrSingle(double a) /// Elementwise standard-normal cumulative probabilities. private static double[] Ndtr(double[] a) { - var scaled = new double[a.Length]; - for (int i = 0; i < a.Length; i++) - scaled[i] = a[i] / Sqrt2; - var erf = Erf.ComputeArray(scaled); var result = new double[a.Length]; - for (int i = 0; i < a.Length; i++) - result[i] = 0.5 + 0.5 * erf[i]; + Ndtr(a, result); return result; } + /// + /// into a caller's buffer, with its two temporaries rented from the shared pool + /// (Rustuna plan 13, item S6): the same per-element arithmetic, and the vectorized-erf switch still sees the + /// logical length, because the rented buffers are passed on as spans of exactly 's length. + /// + /// The standardized arguments. + /// Receives one probability per element of ; at least as long. + /// is shorter than . + private static void Ndtr(ReadOnlySpan a, Span result) + { + int n = a.Length; + double[] scaledBuffer = System.Buffers.ArrayPool.Shared.Rent(n); + double[] erfBuffer = System.Buffers.ArrayPool.Shared.Rent(n); + try + { + var scaled = new Span(scaledBuffer, 0, n); + for (int i = 0; i < n; i++) + scaled[i] = a[i] / Sqrt2; + var erf = new Span(erfBuffer, 0, n); + Erf.ComputeArray(scaled, erf); + for (int i = 0; i < n; i++) + result[i] = 0.5 + 0.5 * erf[i]; + } + finally + { + // Returned on every path: a rented buffer that is never returned only costs the pool, but this keeps the + // pool's buckets warm for the next call of the same size. + System.Buffers.ArrayPool.Shared.Return(scaledBuffer); + System.Buffers.ArrayPool.Shared.Return(erfBuffer); + } + } + /// Python _log_ndtr_single(a). /// The first numerical operand; its dimensions must agree with the operation described in the summary. /// The log standard-normal cumulative probability, evaluated stably in the tails. @@ -111,44 +138,75 @@ private static double LogNdtrSingle(double a) /// The standardized upper truncation bound for each interval. /// The log standard-normal probability mass inside each corresponding interval. internal static double[] LogGaussMass(double[] a, double[] b) + { + var outArr = new double[a.Length]; + LogGaussMass(a, b, outArr); + return outArr; + } + + /// + /// into a caller's buffer, with every temporary rented from the + /// shared pool (Rustuna plan 13, item S6). TPE calls it with tens of thousands of elements per estimator, and the + /// array version's temporaries landed on the large-object heap each time. + /// + /// The standardized lower truncation bound for each interval. + /// The standardized upper truncation bound for each interval; as long as . + /// Receives one log mass per interval; at least as long as , every element written. + /// or is shorter than . + /// + /// Same values as the array version, element by element: the central elements are gathered in index order into + /// buffers of exactly the central count, so + /// picks its exact or vectorized path from the same count as before. + /// + internal static void LogGaussMass(ReadOnlySpan a, ReadOnlySpan b, Span result) { int n = a.Length; - var outArr = new double[n]; // case_left = b <= 0; case_right = a > 0; case_central = ~(left | right). NumPy assigns // left, then right (overwriting), then central — so a > 0 wins the left/right overlap. - var centralIdx = new System.Collections.Generic.List(); - for (int i = 0; i < n; i++) + int[] centralBuffer = System.Buffers.ArrayPool.Shared.Rent(Math.Max(n, 1)); + double[] scratch = System.Buffers.ArrayPool.Shared.Rent(Math.Max(4 * n, 1)); + try { - if (!(a[i] > 0.0) && !(b[i] <= 0.0)) - centralIdx.Add(i); - } + int central = 0; + for (int i = 0; i < n; i++) + { + if (!(a[i] > 0.0) && !(b[i] <= 0.0)) + centralBuffer[central++] = i; + } - if (centralIdx.Count > 0) - { - var ac = new double[centralIdx.Count]; - var negB = new double[centralIdx.Count]; - for (int k = 0; k < centralIdx.Count; k++) + if (central > 0) { - ac[k] = a[centralIdx[k]]; - negB[k] = -b[centralIdx[k]]; + // Four views of one rented buffer, each exactly the central count long. + var ac = new Span(scratch, 0, central); + var negB = new Span(scratch, central, central); + var ndtrAc = new Span(scratch, 2 * central, central); + var ndtrNegB = new Span(scratch, 3 * central, central); + for (int k = 0; k < central; k++) + { + ac[k] = a[centralBuffer[k]]; + negB[k] = -b[centralBuffer[k]]; + } + Ndtr(ac, ndtrAc); + Ndtr(negB, ndtrNegB); + for (int k = 0; k < central; k++) + result[centralBuffer[k]] = MathUtil.Log1p(-ndtrAc[k] - ndtrNegB[k]); } - var ndtrAc = Ndtr(ac); - var ndtrNegB = Ndtr(negB); - for (int k = 0; k < centralIdx.Count; k++) - outArr[centralIdx[k]] = MathUtil.Log1p(-ndtrAc[k] - ndtrNegB[k]); - } - for (int i = 0; i < n; i++) + for (int i = 0; i < n; i++) + { + if (a[i] > 0.0) + // mass_case_right(a, b) = mass_case_left(-b, -a). + result[i] = LogDiff(LogNdtrSingle(-a[i]), LogNdtrSingle(-b[i])); + else if (b[i] <= 0.0) + result[i] = LogDiff(LogNdtrSingle(b[i]), LogNdtrSingle(a[i])); + } + } + finally { - if (a[i] > 0.0) - // mass_case_right(a, b) = mass_case_left(-b, -a). - outArr[i] = LogDiff(LogNdtrSingle(-a[i]), LogNdtrSingle(-b[i])); - else if (b[i] <= 0.0) - outArr[i] = LogDiff(LogNdtrSingle(b[i]), LogNdtrSingle(a[i])); + System.Buffers.ArrayPool.Shared.Return(centralBuffer); + System.Buffers.ArrayPool.Shared.Return(scratch); } - - return outArr; } // ------------------------------------------------------------------------------------------ diff --git a/src/OptunaSharp.Core/Samplers/QMCSampler.cs b/src/OptunaSharp.Core/Samplers/QMCSampler.cs index 22ad709..ce18943 100644 --- a/src/OptunaSharp.Core/Samplers/QMCSampler.cs +++ b/src/OptunaSharp.Core/Samplers/QMCSampler.cs @@ -87,12 +87,38 @@ public QMCSampler( /// public override void ReseedRng() => _independentSampler.ReseedRng(); + /// + /// The study's lowest-numbered COMPLETE or PRUNED trial, or null when there is none: what Optuna reads as + /// sorted(past_trials, key=lambda t: t.number)[0] after + /// study._get_trials(deepcopy=False, states=(COMPLETE, PRUNED), use_cache=True). + /// + /// The study being sampled. + /// The first finished-and-suggested trial in number order, or null. + /// + /// Reads through and stops at the first match (Rustuna plan 13, item S3): + /// the history is in number order, so the first match is the minimum. On an in-memory storage that read is the + /// zero-copy view and the first trial is usually the answer, so an ask costs O(1) instead of a copy and a sort + /// of the whole history; elsewhere it is Optuna's read, cache included. + /// + private static FrozenTrial? FirstSuggestedTrial(Study study) + { + foreach (FrozenTrial past in study.GetTrialsForSampler(useCache: true)) + { + if (Array.IndexOf(SuggestedStates, past.State) >= 0) + return past; + } + return null; + } + /// public override OrderedDict InferRelativeSearchSpace(Study study, FrozenTrial trial) { - var pastTrials = study.GetTrialsInternal(deepcopy: false, states: SuggestedStates, useCache: true); + // Optuna takes past_trials = every COMPLETE or PRUNED trial, then the first of them by number. The history is + // in number order, so that is the first such trial met, and the scan stops there (plan 13, item S3): the copied + // and sorted history made each ask of a long QMC study O(n log n) for one record. + FrozenTrial? firstTrial = FirstSuggestedTrial(study); // The initial trial is sampled by the independent sampler. - if (pastTrials.Count == 0) + if (firstTrial is null) { var pendingTrials = study.GetTrialsInternal( deepcopy: false, states: new[] { TrialState.Running }, useCache: true); @@ -120,7 +146,6 @@ public override OrderedDict InferRelativeSearchSpace(S } // If an initial trial was already made, construct the search space from the first trial. - var firstTrial = pastTrials.OrderBy(t => t.Number).First(); var result = new OrderedDict(); foreach (var (name, dist) in firstTrial.DistributionsDict) result[name] = dist; @@ -131,7 +156,8 @@ public override OrderedDict InferRelativeSearchSpace(S public override object? SampleIndependent( Study study, FrozenTrial trial, string paramName, BaseDistribution paramDistribution) { - if (study.GetTrialsInternal(deepcopy: false, states: SuggestedStates, useCache: true).Count > 0) + // Optuna asks whether any trial is COMPLETE or PRUNED; the first one found answers it (plan 13, item S3). + if (FirstSuggestedTrial(study) is not null) { if (_warnIndependentSampling) LogIndependentSampling(trial, paramName); diff --git a/src/OptunaSharp.Core/Samplers/TPESampler.cs b/src/OptunaSharp.Core/Samplers/TPESampler.cs index 02e6bc4..2fa1a5b 100644 --- a/src/OptunaSharp.Core/Samplers/TPESampler.cs +++ b/src/OptunaSharp.Core/Samplers/TPESampler.cs @@ -55,6 +55,16 @@ public class TPESampler : BaseSampler private readonly bool _constantLiar; private readonly Func>? _constraintsFunc; + /// + /// The last below/above split, reused while the same records are split again (plan 13, item S5). Initialized + /// here, so the public constructors and the per-worker copy constructor each give an instance its own: the cache + /// is guarded by this sampler's monitor, which a worker instance does not share. + /// + private readonly TpeSplitCache _splitCache = new(); + + /// The split cache, for the tests that count how many splits were computed rather than reused. + internal TpeSplitCache SplitCache => _splitCache; + /// Deprecated (v4.3.0). Falls back to true. /// Deprecated (v4.9.0). Falls back to 1.0. /// Deprecated (v4.9.0). Falls back to true. @@ -400,24 +410,42 @@ public override OrderedDict InferRelativeSearchSpace(S private OrderedDict GetInternalRepr( IReadOnlyList trials, OrderedDict searchSpace, Study study) { - var values = new Dictionary>(); - foreach (var name in searchSpace.Keys) - values[name] = new List(); + // One column per search-space parameter, by position, sized for every trial up front (plan 13, item S6): the + // per-trial LINQ closure, the dictionary lookup per value and the list regrowth were a fifth of a 1,000-trial + // multi-objective run, which converts the whole history once per parameter. The values and their order are the + // same: every trial in order, each column in search-space order. + int nParams = searchSpace.Count; + var names = new string[nParams]; + var distributions = new BaseDistribution[nParams]; + var columns = new double[nParams][]; + int p = 0; + foreach (var (name, distribution) in searchSpace) + { + names[p] = name; + distributions[p] = distribution; + columns[p] = new double[trials.Count]; + p++; + } + int rows = 0; foreach (var trial in trials) { var @params = GetParams(trial, study); // search_space.keys() <= params.keys(): the trial must carry every search-space param. - if (searchSpace.Keys.All(k => @params.ContainsKey(k))) + bool carriesAll = true; + for (int i = 0; i < nParams && carriesAll; i++) + carriesAll = @params.ContainsKey(names[i]); + if (carriesAll) { - foreach (var (name, distribution) in searchSpace) - values[name].Add(distribution.ToInternalRepr(@params[name])); + for (int i = 0; i < nParams; i++) + columns[i][rows] = distributions[i].ToInternalRepr(@params[names[i]]); + rows++; } } - var result = new OrderedDict(searchSpace.Count); - foreach (var name in searchSpace.Keys) - result[name] = values[name].ToArray(); + var result = new OrderedDict(nParams); + for (int i = 0; i < nParams; i++) + result[names[i]] = rows == trials.Count ? columns[i] : columns[i][..rows]; return result; } @@ -433,11 +461,24 @@ private OrderedDict GetInternalRepr( ? new[] { TrialState.Complete, TrialState.Pruned, TrialState.Running } : new[] { TrialState.Complete, TrialState.Pruned }; bool useCache = !_constantLiar; - var trials = study.GetTrialsInternal(deepcopy: false, states: states, useCache: useCache); + List trials; + if (_constantLiar && study.Storage.HasZeroCopyTrialsView) + { + // For constant_liar, filter out the current trial: here by never reading its record (plan 13, item S3). The + // read below returns the stored records themselves, and a running record that is read must be copied before + // its next write (InMemoryStorage, item S4); reading the current trial's record only to drop it made every + // suggestion of an independent-TPE trial copy that trial. The zero-copy view yields the same records in the + // same (trial number) order as GetAllTrials(deepcopy: false, states), minus the current trial. + trials = ReadOtherTrials(study, trial.Number, states); + } + else + { + trials = study.GetTrialsInternal(deepcopy: false, states: states, useCache: useCache); - if (_constantLiar) - // For constant_liar, filter out the current trial. - trials = trials.Where(t => trial.Number != t.Number).ToList(); + if (_constantLiar) + // For constant_liar, filter out the current trial. + trials = trials.Where(t => trial.Number != t.Number).ToList(); + } // Divide data into below and above. if (_gamma is null) @@ -447,7 +488,9 @@ private OrderedDict GetInternalRepr( int n = trials.Count(t => t.State != TrialState.Running); // Ignore running trials. - var (belowTrials, aboveTrials) = TpeTrialSplit.SplitTrials(study, trials, _gamma(n)); + // The same split as TpeTrialSplit.SplitTrials, reused while the same records are split again: every parameter + // of an independent-TPE trial splits the same history (plan 13, item S5; see TpeSplitCache). + var (belowTrials, aboveTrials) = _splitCache.Split(study, trials, _gamma(n)); var mpeBelow = BuildParzenEstimator(study, searchSpace, belowTrials, handleBelow: true); var mpeAbove = BuildParzenEstimator(study, searchSpace, aboveTrials, handleBelow: false); @@ -463,6 +506,37 @@ private OrderedDict GetInternalRepr( return result; } + /// + /// The constant-liar history read for a storage whose + /// is zero-copy: + /// every trial in by trial number, except trial , whose + /// record is never read. + /// + /// The study being sampled; its storage has a zero-copy view. + /// The number of the trial being sampled, which constant liar leaves out. + /// The states to keep. + /// The records GetAllTrials(deepcopy: false, states) returns, without the current trial's. + /// + /// In-memory trial numbers are positions in the study's trial list, which is the view's index, so skipping a + /// position skips the trial without reading it. Each other position is read once, as the view reads it: a finished + /// record without the lock, a running one under it. Another worker's running trial is read, and so copied before + /// that worker's next write, as before: constant liar uses those records. + /// + private static List ReadOtherTrials(Study study, int currentNumber, TrialState[] states) + { + IReadOnlyList view = study.Storage.GetTrialsPrefixView(study.StudyId); + var result = new List(view.Count); + for (int i = 0; i < view.Count; i++) + { + if (i == currentNumber) + continue; + FrozenTrial other = view[i]; + if (Array.IndexOf(states, other.State) >= 0) + result.Add(other); + } + return result; + } + /// Python _build_parzen_estimator. /// Study supplying trial history, objective directions, and storage. /// Ordered parameter distributions defining the coordinates used by this operation. diff --git a/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs b/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs index 7a81507..bbe7691 100644 --- a/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs +++ b/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs @@ -316,7 +316,7 @@ private static double[] ComputeSigmas( // sorted_indices = np.argsort(mus). Stable — exact for distinct mus. NumPy 2.x sorts float64 // with x86-simd-sort, whose tie order is not portably reproducible; see the class remark for // the (documented) divergence this leaves on repeated numerical observations. - var sortedIndices = Enumerable.Range(0, n).OrderBy(i => mus[i]).ToArray(); + var sortedIndices = StableArgsort(mus); // sorted_mus_with_endpoints = [low, *sorted_mus, high]. var swe = new double[n + 2]; @@ -358,6 +358,44 @@ private static double[] ComputeSigmas( return sigmas; } + /// + /// The stable argsort uses: the permutation + /// Enumerable.Range(0, n).OrderBy(i => values[i]) returns, with ties in index order, computed by + /// and an index sort of each run of equal keys. + /// + /// The values to order; not modified. + /// The indices of in ascending order of value, equal values by index. + /// + /// Why not the LINQ sort it replaces (Rustuna plan 13, item S6): it compares through a delegate and allocated + /// its key buffers per call, about 0.8 s of a 1,000-trial, 40-parameter TPE run. The result is the same + /// permutation: orders doubles as + /// does (NaN first, -0.0 equal to 0.0), which is what the LINQ sort's + /// default comparer used, and its only freedom, the order inside a run of equal keys, is undone by sorting each + /// run's indices. + /// + internal static int[] StableArgsort(double[] values) + { + int n = values.Length; + var keys = (double[])values.Clone(); + var indices = new int[n]; + for (int i = 0; i < n; i++) + indices[i] = i; + Array.Sort(keys, indices); + + int runStart = 0; + for (int i = 1; i <= n; i++) + { + // A run ends where the key changes by CompareTo, which groups NaNs together and -0.0 with 0.0. + if (i == n || keys[i].CompareTo(keys[runStart]) != 0) + { + if (i - runStart > 1) + Array.Sort(indices, runStart, i - runStart); + runStart = i; + } + } + return indices; + } + /// Python _ParzenEstimator._call_weights_func. /// Callback returning one nonnegative finite weight per observation. /// Number of vector entries to produce. diff --git a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs index d846465..e4ccc06 100644 --- a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs +++ b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs @@ -340,184 +340,302 @@ internal double[] LogPdf(double[,] x) int nSamples = x.GetLength(0); int dim = x.GetLength(1); int nKernels = Weights.Length; - var weightedLogPdf = new double[nSamples, nKernels]; + // weighted_log_pdf, (n_samples, n_kernels) row-major in a rented buffer (plan 13, item S6): at TPE's 24 candidates + // and a thousand kernels it is 192 KB, a large-object-heap allocation per call, which kept background GCs busy + // and made the sampler wait for them. Every element is cleared, then accumulated in the same order as before. + double[] weightedLogPdf = System.Buffers.ArrayPool.Shared.Rent(Math.Max(nSamples * nKernels, 1)); + try + { + Array.Clear(weightedLogPdf, 0, nSamples * nKernels); - var contDists = new List(); - var contCols = new List(); + var contDists = new List(); + var contCols = new List(); - for (int i = 0; i < dim; i++) - { - switch (Distributions[i]) + for (int i = 0; i < dim; i++) { - case BatchedCategoricalDistributions cat: + switch (Distributions[i]) { - // xi = x[:, i, None, None].astype(int64); += log(take_along_axis(weights, xi))[..., 0]. - for (int s = 0; s < nSamples; s++) + case BatchedCategoricalDistributions cat: { - long xi = (long)x[s, i]; - for (int k = 0; k < nKernels; k++) - weightedLogPdf[s, k] += Math.Log(cat.Weights[k, xi]); + // xi = x[:, i, None, None].astype(int64); += log(take_along_axis(weights, xi))[..., 0]. + for (int s = 0; s < nSamples; s++) + { + long xi = (long)x[s, i]; + for (int k = 0; k < nKernels; k++) + weightedLogPdf[s * nKernels + k] += Math.Log(cat.Weights[k, xi]); + } + break; } - break; - } - case BatchedNumericalDistributions num when num.Step == 0.0: - // Continuous kind: collected and processed together after the loop. - contDists.Add(num); - contCols.Add(i); - break; + case BatchedNumericalDistributions num when num.Step == 0.0: + // Continuous kind: collected and processed together after the loop. + contDists.Add(num); + contCols.Add(i); + break; - case BatchedNumericalDistributions dtn: - { - // Discrete kind (truncated normal or log-normal). - bool isLog = dtn.IsLog; - double step = dtn.Step; - var xcol = new double[nSamples]; - for (int s = 0; s < nSamples; s++) - xcol[s] = x[s, i]; - - var (xiUniq, xiInv) = UniqueInverse1d(xcol); - var (muUniq, sigmaUniq, msInv) = UniqueInverse2d(dtn.Mu, dtn.Sigma); - int nXi = xiUniq.Length; - int nMs = muUniq.Length; - - var a = new double[nXi * nMs]; - var b = new double[nXi * nMs]; - for (int p = 0; p < nXi; p++) + case BatchedNumericalDistributions dtn: { - double left = isLog ? Math.Log(xiUniq[p] - step / 2.0) : xiUniq[p] - step / 2.0; - double right = isLog ? Math.Log(xiUniq[p] + step / 2.0) : xiUniq[p] + step / 2.0; - for (int qi = 0; qi < nMs; qi++) + // Discrete kind (truncated normal or log-normal). + bool isLog = dtn.IsLog; + double step = dtn.Step; + var xcol = new double[nSamples]; + for (int s = 0; s < nSamples; s++) + xcol[s] = x[s, i]; + + var (xiUniq, xiInv) = UniqueInverse1d(xcol); + var (muUniq, sigmaUniq, msInv) = UniqueInverse2d(dtn.Mu, dtn.Sigma); + int nXi = xiUniq.Length; + int nMs = muUniq.Length; + + var a = new double[nXi * nMs]; + var b = new double[nXi * nMs]; + for (int p = 0; p < nXi; p++) { - a[p * nMs + qi] = (left - muUniq[qi]) / sigmaUniq[qi]; - b[p * nMs + qi] = (right - muUniq[qi]) / sigmaUniq[qi]; + double left = isLog ? Math.Log(xiUniq[p] - step / 2.0) : xiUniq[p] - step / 2.0; + double right = isLog ? Math.Log(xiUniq[p] + step / 2.0) : xiUniq[p] + step / 2.0; + for (int qi = 0; qi < nMs; qi++) + { + a[p * nMs + qi] = (left - muUniq[qi]) / sigmaUniq[qi]; + b[p * nMs + qi] = (right - muUniq[qi]) / sigmaUniq[qi]; + } } - } - var logMass = LogGaussMassUnique(a, b); // shape (nXi, nMs), row-major. - for (int s = 0; s < nSamples; s++) - { - int row = xiInv[s] * nMs; - for (int k = 0; k < nKernels; k++) - weightedLogPdf[s, k] += logMass[row + msInv[k]]; - } + var logMass = LogGaussMassUnique(a, b); // shape (nXi, nMs), row-major. + for (int s = 0; s < nSamples; s++) + { + int row = xiInv[s] * nMs; + for (int k = 0; k < nKernels; k++) + weightedLogPdf[s * nKernels + k] += logMass[row + msInv[k]]; + } - var normLow = new double[nMs]; - var normHigh = new double[nMs]; - for (int qi = 0; qi < nMs; qi++) - { - normLow[qi] = (dtn.AdaptedLow - muUniq[qi]) / sigmaUniq[qi]; - normHigh[qi] = (dtn.AdaptedHigh - muUniq[qi]) / sigmaUniq[qi]; - } - var norm = TruncNorm.LogGaussMass(normLow, normHigh); - for (int s = 0; s < nSamples; s++) - { - for (int k = 0; k < nKernels; k++) - weightedLogPdf[s, k] -= norm[msInv[k]]; + var normLow = new double[nMs]; + var normHigh = new double[nMs]; + for (int qi = 0; qi < nMs; qi++) + { + normLow[qi] = (dtn.AdaptedLow - muUniq[qi]) / sigmaUniq[qi]; + normHigh[qi] = (dtn.AdaptedHigh - muUniq[qi]) / sigmaUniq[qi]; + } + var norm = TruncNorm.LogGaussMass(normLow, normHigh); + for (int s = 0; s < nSamples; s++) + { + for (int k = 0; k < nKernels; k++) + weightedLogPdf[s * nKernels + k] -= norm[msInv[k]]; + } + break; } - break; - } - default: - throw new InvalidOperationException("Unknown batched distribution kind."); + default: + throw new InvalidOperationException("Unknown batched distribution kind."); + } } - } - if (contDists.Count > 0) - AddContinuousLogPdf(x, weightedLogPdf, contDists, contCols); + if (contDists.Count > 0) + AddContinuousLogPdf(x, weightedLogPdf, contDists, contCols); - // weighted_log_pdf += np.log(self.weights[np.newaxis]). - for (int s = 0; s < nSamples; s++) - { + // weighted_log_pdf += np.log(self.weights[np.newaxis]). Each kernel's log weight is the same for every sample, + // so it is taken once per kernel (plan 13, item S6); Math.Log is deterministic, so the sums keep their bits. + var logWeights = new double[nKernels]; for (int k = 0; k < nKernels; k++) - weightedLogPdf[s, k] += Math.Log(Weights[k]); - } + logWeights[k] = Math.Log(Weights[k]); + for (int s = 0; s < nSamples; s++) + { + for (int k = 0; k < nKernels; k++) + weightedLogPdf[s * nKernels + k] += logWeights[k]; + } - // np.log(np.exp(weighted_log_pdf - max_[:, None]).sum(axis=1)) + max_, with -inf max_ -> 0. - var result = new double[nSamples]; - var expBuf = new double[nKernels]; - for (int s = 0; s < nSamples; s++) + // np.log(np.exp(weighted_log_pdf - max_[:, None]).sum(axis=1)) + max_, with -inf max_ -> 0. + var result = new double[nSamples]; + var expBuf = new double[nKernels]; + for (int s = 0; s < nSamples; s++) + { + double max = double.NegativeInfinity; + for (int k = 0; k < nKernels; k++) + max = Math.Max(max, weightedLogPdf[s * nKernels + k]); + if (double.IsNegativeInfinity(max)) + max = 0.0; + for (int k = 0; k < nKernels; k++) + expBuf[k] = Math.Exp(weightedLogPdf[s * nKernels + k] - max); + result[s] = Math.Log(MathUtil.PairwiseSum(expBuf)) + max; + } + return result; + } + finally { - double max = double.NegativeInfinity; - for (int k = 0; k < nKernels; k++) - max = Math.Max(max, weightedLogPdf[s, k]); - if (double.IsNegativeInfinity(max)) - max = 0.0; - for (int k = 0; k < nKernels; k++) - expBuf[k] = Math.Exp(weightedLogPdf[s, k] - max); - result[s] = Math.Log(MathUtil.PairwiseSum(expBuf)) + max; + System.Buffers.ArrayPool.Shared.Return(weightedLogPdf); } - return result; } /// /// The continuous branch of log_pdf: _truncnorm.logpdf over all continuous dims, /// summed over dims and added to . The Gaussian-mass grid is - /// computed once over the (n_kernels, n_cont) standardized bounds, exactly as NumPy does, - /// so the array-size threshold is exercised identically. + /// computed once over the n_kernels × n_cont standardized bounds, as NumPy does, so the + /// array-size threshold is exercised identically. /// /// Input numerical coordinates or observation rows. - /// Log densities including each mixture component weight. + /// + /// Log densities including each mixture component weight, (n_samples, n_kernels) row-major; at least + /// n_samples × n_kernels long (a rented buffer may be longer). + /// /// Continuous numerical coordinate distributions. /// Original coordinate indices of continuous numerical distributions. + /// + /// + /// Fused, same bits (Rustuna plan 13, item S6). Everything that depends only on a (kernel, dim) + /// pair is computed once per call instead of once per candidate: the standardized bounds, their log + /// Gaussian mass and log(sigma), which the per-candidate loop used to take 24 times per pair (more + /// than half of a 1,000-trial, 40-parameter TPE run). Each value is the same expression on the same + /// operands, and Math.Log is deterministic, so it has the same bits. + /// + /// + /// The grids are laid out by dimension ([c * n_kernels + k]) so the kernel loop is contiguous. + /// is elementwise, and the vectorized-erf switch depends only + /// on how many elements it gets (see ), so evaluating the same elements in + /// another order changes no element. + /// + /// + /// adds one dimension's term to every kernel's sum, dimension after + /// dimension, so each kernel's sum is still ((0 + t0) + t1) + … in dimension order; vector lanes + /// hold different kernels and never mix. .NET never contracts a multiply and an add into a fused + /// multiply-add on its own, so a lane computes exactly what the scalar loop computes. + /// + /// private void AddContinuousLogPdf( - double[,] x, double[,] weightedLogPdf, + double[,] x, double[] weightedLogPdf, List contDists, List contCols) { int nSamples = x.GetLength(0); int nKernels = Weights.Length; int nCont = contDists.Count; - - // x_cont[c][s] = log(x[s, col]) if is_log else x[s, col]. - var xCont = new double[nCont][]; + int gridSize = nKernels * nCont; + + // Every buffer below is fully written before it is read, so none is zeroed first: zeroing the grids was a tenth + // of a 1,000-trial, 40-parameter TPE run, and allocating them put four large-object-heap arrays per call on the + // GC. The grids are rented, and every use goes through a span of the logical length: LogGaussMass and the + // vectorized-erf switch depend on the length they are given, never on a rented buffer's size. + // x_cont = log(x[:, col]) where is_log else x[:, col], one row per sample ([s * nCont + c]). + var xCont = GC.AllocateUninitializedArray(nSamples * nCont); for (int c = 0; c < nCont; c++) { bool isLog = contDists[c].IsLog; int col = contCols[c]; - var arr = new double[nSamples]; for (int s = 0; s < nSamples; s++) - arr[s] = isLog ? Math.Log(x[s, col]) : x[s, col]; - xCont[c] = arr; + xCont[s * nCont + c] = isLog ? Math.Log(x[s, col]) : x[s, col]; } - // Standardized bounds a, b over (n_kernels, n_cont), raveled row-major (k outer, c inner). - var aGrid = new double[nKernels * nCont]; - var bGrid = new double[nKernels * nCont]; - for (int k = 0; k < nKernels; k++) + // Standardized bounds a, b, log(sigma) and the log Gaussian mass per (kernel, dim): one rented buffer cut into + // four grids, each one row per dim over the kernels. + double[] gridBuffer = System.Buffers.ArrayPool.Shared.Rent(4 * gridSize); + try { + var aGrid = new Span(gridBuffer, 0, gridSize); + var bGrid = new Span(gridBuffer, gridSize, gridSize); + var logSigma = new Span(gridBuffer, 2 * gridSize, gridSize); + var logGaussMass = new Span(gridBuffer, 3 * gridSize, gridSize); for (int c = 0; c < nCont; c++) { var nd = contDists[c]; - double sigma = nd.Sigma[k]; - aGrid[k * nCont + c] = (nd.AdaptedLow - nd.Mu[k]) / sigma; - bGrid[k * nCont + c] = (nd.AdaptedHigh - nd.Mu[k]) / sigma; + // AdaptedLow / AdaptedHigh recompute a logarithm per access on the log kinds: the same value every time. + double adaptedLow = nd.AdaptedLow, adaptedHigh = nd.AdaptedHigh; + double[] mu = nd.Mu, sigma = nd.Sigma; + int row = c * nKernels; + for (int k = 0; k < nKernels; k++) + { + aGrid[row + k] = (adaptedLow - mu[k]) / sigma[k]; + bGrid[row + k] = (adaptedHigh - mu[k]) / sigma[k]; + logSigma[row + k] = Math.Log(sigma[k]); + } } - } - var logGaussMass = TruncNorm.LogGaussMass(aGrid, bGrid); + TruncNorm.LogGaussMass(aGrid, bGrid, logGaussMass); - for (int s = 0; s < nSamples; s++) - { - for (int k = 0; k < nKernels; k++) + var sums = GC.AllocateUninitializedArray(nKernels); + for (int s = 0; s < nSamples; s++) { - double sum = 0.0; + // Each kernel's sum starts at +0.0 for every candidate, as the scalar loop's `double sum = 0.0` did. + Array.Clear(sums); for (int c = 0; c < nCont; c++) { var nd = contDists[c]; - double mu = nd.Mu[k]; - double sigma = nd.Sigma[k]; - int gi = k * nCont + c; - double xstd = (xCont[c][s] - mu) / sigma; - double val = TruncNorm.NormLogPdf(xstd) - logGaussMass[gi] - Math.Log(sigma); - // np.select([a == b, (x < a) | (x > b)], [nan, -inf], default=out). - if (aGrid[gi] == bGrid[gi]) - val = double.NaN; - else if (xstd < aGrid[gi] || xstd > bGrid[gi]) - val = double.NegativeInfinity; - sum += val; + int row = c * nKernels; + AccumulateDimension( + xCont[s * nCont + c], nd.Mu, nd.Sigma, + aGrid.Slice(row, nKernels), bGrid.Slice(row, nKernels), + logGaussMass.Slice(row, nKernels), logSigma.Slice(row, nKernels), + sums); } - weightedLogPdf[s, k] += sum; + for (int k = 0; k < nKernels; k++) + weightedLogPdf[s * nKernels + k] += sums[k]; } } + finally + { + System.Buffers.ArrayPool.Shared.Return(gridBuffer); + } + } + + /// + /// Adds one continuous dimension's truncated-normal log density at to every kernel's sum: + /// (norm_logpdf((x - mu) / sigma) - log_gauss_mass) - log(sigma), NaN where the kernel's standardized + /// interval is empty (a == b) and -inf where x falls outside it, as + /// np.select([a == b, (x < a) | (x > b)], [nan, -inf], default=out). + /// + /// The candidate's coordinate in this dimension, log-transformed already for the log kinds. + /// The kernels' centers in this dimension. + /// The kernels' standard deviations in this dimension. + /// The kernels' standardized lower bounds. + /// The kernels' standardized upper bounds. + /// The kernels' log Gaussian mass between and . + /// The kernels' log(sigma). + /// One running sum per kernel; this dimension's term is added to each. + /// + /// The vector path computes every lane with the scalar path's operations in the scalar path's order: the same + /// division (never a multiplication by the reciprocal), the same subtraction order, the NaN and -inf + /// overrides applied last. The only freedom it takes is the sign of zero in -(z * z), which the + /// subtraction of the normal's log constant right after removes. The tail that does not fill a vector takes + /// the scalar path. + /// + private static void AccumulateDimension( + double x, double[] mu, double[] sigma, + ReadOnlySpan a, ReadOnlySpan b, ReadOnlySpan logGaussMass, ReadOnlySpan logSigma, + double[] sums) + { + int n = sums.Length; + int k = 0; + if (System.Numerics.Vector.IsHardwareAccelerated && n >= System.Numerics.Vector.Count) + { + int width = System.Numerics.Vector.Count; + var xv = new System.Numerics.Vector(x); + var two = new System.Numerics.Vector(2.0); + var logC = new System.Numerics.Vector(TruncNorm.NormPdfLogC); + var nan = new System.Numerics.Vector(double.NaN); + var negativeInfinity = new System.Numerics.Vector(double.NegativeInfinity); + for (; k <= n - width; k += width) + { + var muV = new System.Numerics.Vector(mu, k); + var sigmaV = new System.Numerics.Vector(sigma, k); + var aV = new System.Numerics.Vector(a[k..]); + var bV = new System.Numerics.Vector(b[k..]); + var xstd = (xv - muV) / sigmaV; + var normLogPdf = -(xstd * xstd) / two - logC; + var val = (normLogPdf - new System.Numerics.Vector(logGaussMass[k..])) + - new System.Numerics.Vector(logSigma[k..]); + val = System.Numerics.Vector.ConditionalSelect( + System.Numerics.Vector.LessThan(xstd, aV) | System.Numerics.Vector.GreaterThan(xstd, bV), + negativeInfinity, val); + val = System.Numerics.Vector.ConditionalSelect(System.Numerics.Vector.Equals(aV, bV), nan, val); + (new System.Numerics.Vector(sums, k) + val).CopyTo(sums, k); + } + } + + for (; k < n; k++) + { + double xstd = (x - mu[k]) / sigma[k]; + double val = TruncNorm.NormLogPdf(xstd) - logGaussMass[k] - logSigma[k]; + if (a[k] == b[k]) + val = double.NaN; + else if (xstd < a[k] || xstd > b[k]) + val = double.NegativeInfinity; + sums[k] += val; + } } /// NumPy searchsorted(cdf, v, side='right') for an ascending array (bisect right). diff --git a/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs b/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs index 18e6311..ff8172e 100644 --- a/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs +++ b/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs @@ -99,8 +99,13 @@ public SearchSpaceGroup Calculate(Study study, bool useCache = false) ? new[] { TrialState.Complete, TrialState.Pruned } : new[] { TrialState.Complete }; - foreach (var trial in study.GetTrialsInternal(deepcopy: false, states: statesOfInterest, useCache: useCache)) - _searchSpace.AddDistributions(trial.DistributionsDict); + // The same trials in the same (number) order as study._get_trials(deepcopy=False, states, use_cache), filtered + // here instead of in a copied list: on an in-memory storage the read is the zero-copy view (plan 13, item S3). + foreach (var trial in study.GetTrialsForSampler(useCache)) + { + if (System.Array.IndexOf(statesOfInterest, trial.State) >= 0) + _searchSpace.AddDistributions(trial.DistributionsDict); + } return _searchSpace.DeepCopy(); } diff --git a/test/OptunaSharp.Tests.Interop/NsgaiiSamplerParityTests.cs b/test/OptunaSharp.Tests.Interop/NsgaiiSamplerParityTests.cs index 437295c..167610a 100644 --- a/test/OptunaSharp.Tests.Interop/NsgaiiSamplerParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/NsgaiiSamplerParityTests.cs @@ -318,4 +318,45 @@ public void ConstrainedConstraintsFunc_FullState_IsBitIdentical(int seed) " return (x * x + y * y, x - y)\n" + "study.optimize(_obj, n_trials=20)\n"); } + + /// + /// Plan 13 gate (docs/plans/13-rustuna-derived-performance.md, phase 0): the default NSGA-II over 40 float + /// parameters for 2,000 trials (40 generations of 50), the workload whose suggestions item S4 now writes in place. + /// Every trial's full state, generations and parent caches included, is bit-identical to live Optuna's. + /// + [TestMethod] + [TestCategory("RequiresPython")] + [TestCategory("Slow")] + public void LongRun_40Parameters_2000Trials_FullState_IsBitIdentical() + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "nsga", sampler: new NSGAIISampler(seed: 0), + directions: new[] { StudyDirection.Minimize, StudyDirection.Minimize }); + csStudy.Optimize(t => + { + double a = 0, b = 0; + for (int i = 0; i < 40; i++) + { + double v = t.SuggestFloat($"x{i}", -10, 10); + a += v * v; + b += (v - 2) * (v - 2); + } + return new[] { a, b }; + }, nTrials: 2000); + + AssertFullParity(csStudy, + "study = optuna.create_study(study_name='nsga', directions=['minimize', 'minimize'], " + + "sampler=optuna.samplers.NSGAIISampler(seed=0))\n" + + "def _obj(t):\n" + + " a = 0.0\n" + + " b = 0.0\n" + + " for i in range(40):\n" + + " v = t.suggest_float(f'x{i}', -10, 10)\n" + + " a += v * v\n" + + " b += (v - 2) * (v - 2)\n" + + " return a, b\n" + + "study.optimize(_obj, n_trials=2000)\n"); + } } diff --git a/test/OptunaSharp.Tests.Interop/OptunaHubBridgeParityTests.cs b/test/OptunaSharp.Tests.Interop/OptunaHubBridgeParityTests.cs index f73de1e..17c85a8 100644 --- a/test/OptunaSharp.Tests.Interop/OptunaHubBridgeParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/OptunaHubBridgeParityTests.cs @@ -42,7 +42,10 @@ public void BenchmarkBridge_ToyProblem_ExposesSpaceAndOptimizes() Assert.IsInstanceOfType(problem.SearchSpace["n"], typeof(IntDistribution)); Assert.IsInstanceOfType(problem.SearchSpace["c"], typeof(CategoricalDistribution)); - var study = Optuna.CreateStudy(directions: problem.Directions); + // Seeded: an unseeded TPE study draws its seed from OS entropy, and 40 trials then end above the bound + // below in a few percent of runs (one did on 2026-11-09, best 1.018; eight reruns passed). The test is about + // the bridge, not TPE's luck, so the sampler is seeded and the outcome fixed. + var study = Optuna.CreateStudy(directions: problem.Directions, sampler: new OptunaSharp.Samplers.TPESampler(seed: 0)); study.Optimize(problem.SingleObjective, nTrials: 40); Assert.AreEqual(40, study.Trials.Count); diff --git a/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs b/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs index 88e1148..474332c 100644 --- a/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs @@ -326,4 +326,83 @@ public void StartupPhase_UsesRandomSampling_BitExact() for (int i = 0; i < 10; i++) csState.Trials[i].ParamsInternal["x"].Should().Be(pyState.Trials[i].ParamsInternal["x"]); } + + /// + /// Plan 13 gate (docs/plans/13-rustuna-derived-performance.md, phase 0): the default single-objective TPE + /// (multivariate, constant liar) over 40 float parameters for 1,000 trials, the workload whose scoring kernel + /// item S6 rewrote. The whole trajectory stays in lock-step with live Optuna, within the erf tolerance of the + /// other TPE cases, so the fused kernel, the pooled buffers and the sort that replaced LINQ changed nothing a + /// thousand trials of compounding would show. + /// + [TestMethod] + [TestCategory("RequiresPython")] + [TestCategory("Slow")] + public void LongRun_Multivariate40D_1000Trials_TrajectoryMatches() + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: new TPESampler(seed: 0), studyName: "tpe"); + csStudy.Optimize(t => + { + double s = 0; + for (int i = 0; i < 40; i++) + { + double v = t.SuggestFloat($"x{i}", -10, 10); + s += (v - 1) * (v - 1); + } + return s; + }, nTrials: 1000); + + AssertTrajectoryParity(csStudy, + "study = optuna.create_study(study_name='tpe', sampler=optuna.samplers.TPESampler(seed=0))\n" + + "def _obj(t):\n" + + " s = 0.0\n" + + " for i in range(40):\n" + + " v = t.suggest_float(f'x{i}', -10, 10)\n" + + " s += (v - 1) * (v - 1)\n" + + " return s\n" + + "study.optimize(_obj, n_trials=1000)\n"); + } + + /// + /// Plan 13 gate (phase 0): the default two-objective TPE (independent, constant liar) over 40 float parameters + /// for 300 trials, the workload of items S3 (the constant-liar read that skips the current trial), S5 (one split per + /// trial instead of per parameter) and S6. Every trial stays in lock-step with live Optuna within the erf + /// tolerance. + /// + [TestMethod] + [TestCategory("RequiresPython")] + [TestCategory("Slow")] + public void LongRun_MultiObjective40D_300Trials_TrajectoryMatches() + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), sampler: new TPESampler(seed: 0), studyName: "tpe", + directions: new[] { StudyDirection.Minimize, StudyDirection.Minimize }); + csStudy.Optimize(t => + { + double a = 0, b = 0; + for (int i = 0; i < 40; i++) + { + double v = t.SuggestFloat($"x{i}", -10, 10); + a += v * v; + b += (v - 2) * (v - 2); + } + return new[] { a, b }; + }, nTrials: 300); + + AssertTrajectoryParity(csStudy, + "study = optuna.create_study(study_name='tpe', directions=['minimize', 'minimize'], " + + "sampler=optuna.samplers.TPESampler(seed=0))\n" + + "def _obj(t):\n" + + " a = 0.0\n" + + " b = 0.0\n" + + " for i in range(40):\n" + + " v = t.suggest_float(f'x{i}', -10, 10)\n" + + " a += v * v\n" + + " b += (v - 2) * (v - 2)\n" + + " return a, b\n" + + "study.optimize(_obj, n_trials=300)\n"); + } } diff --git a/test/OptunaSharp.Tests/Samplers/TpeExactKernelTests.cs b/test/OptunaSharp.Tests/Samplers/TpeExactKernelTests.cs new file mode 100644 index 0000000..4eed2f1 --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/TpeExactKernelTests.cs @@ -0,0 +1,404 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Internal; +using OptunaSharp.Samplers.Tpe; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S6: the TPE scoring kernel computes per (kernel, +/// dimension) what used to be computed per candidate, runs its kernel loop on vectors, rents its grids, and sorts the +/// kernels' centers without LINQ, all keeping every bit. Each test compares the new code with a straightforward copy +/// of the code it replaced, bit for bit, on inputs chosen to reach every branch: every vector-remainder size, the +/// exact and the vectorized erf paths, log dimensions, candidates outside a kernel's interval, empty intervals, +/// ties and NaNs in the sort. +/// +[TestClass] +public class TpeExactKernelTests +{ + // ------------------------------------------------------------------ the continuous log-pdf + + /// + /// Mixtures of continuous kernels (plain and log), with every kernel count from 1 to three vectors and a few more + /// (so every remainder the scalar tail handles), against the replaced scalar code: every candidate's log density + /// has the same bits. + /// + /// Seeds the random mixtures. + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void ContinuousLogPdf_EveryKernelCount_MatchesTheReplacedScalarCodeToTheBit(int seed) + { + var rng = new Random(seed); + int maxKernels = 3 * System.Numerics.Vector.Count + 5; + foreach (int nCont in new[] { 1, 2, 3, 7, 40 }) + { + for (int nKernels = 1; nKernels <= maxKernels; nKernels++) + AssertSameBits(RandomMixture(rng, nKernels, nCont, logEvery: 3), RandomCandidates(rng, 24, nCont, 0.15)); + } + } + + /// + /// A study-sized mixture (1,000 kernels, 3 and 40 dimensions): the bounds grid has thousands of central + /// elements, so takes Optuna's vectorized erf, + /// reached now through rented buffers cut to the logical length. Same bits. + /// + [TestMethod] + public void ContinuousLogPdf_LargeGrids_UseTheVectorizedErf_AndMatchToTheBit() + { + var rng = new Random(11); + AssertSameBits(RandomMixture(rng, 1000, 3, logEvery: 2), RandomCandidates(rng, 24, 3, 0.1)); + AssertSameBits(RandomMixture(rng, 1000, 40, logEvery: 5), RandomCandidates(rng, 24, 40, 0.1)); + } + + /// + /// A dimension whose bounds coincide (a single-value distribution) gives every kernel an empty interval, so its + /// term is NaN, as np.select makes it; the new kernel yields the same NaN densities. + /// + [TestMethod] + public void ContinuousLogPdf_AnEmptyInterval_IsNaN_AsBefore() + { + var rng = new Random(5); + int nKernels = 2 * System.Numerics.Vector.Count + 1; + var dists = new List + { + RandomDimension(rng, nKernels, log: false), + new BatchedTruncNormDistributions(Fill(nKernels, 0.5), Fill(nKernels, 0.25), 0.5, 0.5), + }; + var mixture = new MixtureOfProductDistribution(RandomWeights(rng, nKernels), dists); + double[,] x = RandomCandidates(rng, 6, 2, 0.0); + for (int s = 0; s < 6; s++) + x[s, 1] = 0.5; + + AssertSameBits(mixture, x); + mixture.LogPdf(x).Should().OnlyContain(v => double.IsNaN(v)); + } + + /// Compares the new with the replaced code, bit for bit. + /// A mixture of continuous dimensions only. + /// The candidates, one row each. + private static void AssertSameBits(MixtureOfProductDistribution mixture, double[,] x) + { + double[] actual = mixture.LogPdf(x); + double[] expected = ReferenceLogPdf(mixture, x); + actual.Length.Should().Be(expected.Length); + for (int s = 0; s < expected.Length; s++) + { + if (double.IsNaN(expected[s])) + double.IsNaN(actual[s]).Should().BeTrue($"candidate {s} was NaN before"); + else + BitConverter.DoubleToInt64Bits(actual[s]).Should().Be(BitConverter.DoubleToInt64Bits(expected[s]), + $"candidate {s}: {actual[s]:R} vs {expected[s]:R} ({mixture.Weights.Length} kernels, {x.GetLength(1)} dims)"); + } + } + + /// + /// The log-pdf as it was computed before item S6, for mixtures of continuous dimensions: per candidate, per kernel, + /// per dimension, with log(sigma) taken in the innermost loop, the bounds grid kernel-major, then the log + /// weights per (candidate, kernel) and the log-sum-exp. + /// + /// A mixture of continuous dimensions only. + /// The candidates, one row each. + /// One log density per candidate. + private static double[] ReferenceLogPdf(MixtureOfProductDistribution mixture, double[,] x) + { + int nSamples = x.GetLength(0), nKernels = mixture.Weights.Length, nCont = mixture.Distributions.Count; + var weighted = new double[nSamples, nKernels]; + var dists = mixture.Distributions.Cast().ToList(); + + var xCont = new double[nCont][]; + for (int c = 0; c < nCont; c++) + { + xCont[c] = new double[nSamples]; + for (int s = 0; s < nSamples; s++) + xCont[c][s] = dists[c].IsLog ? Math.Log(x[s, c]) : x[s, c]; + } + + var aGrid = new double[nKernels * nCont]; + var bGrid = new double[nKernels * nCont]; + for (int k = 0; k < nKernels; k++) + { + for (int c = 0; c < nCont; c++) + { + aGrid[k * nCont + c] = (dists[c].AdaptedLow - dists[c].Mu[k]) / dists[c].Sigma[k]; + bGrid[k * nCont + c] = (dists[c].AdaptedHigh - dists[c].Mu[k]) / dists[c].Sigma[k]; + } + } + double[] logGaussMass = ReferenceLogGaussMass(aGrid, bGrid); + + for (int s = 0; s < nSamples; s++) + { + for (int k = 0; k < nKernels; k++) + { + double sum = 0.0; + for (int c = 0; c < nCont; c++) + { + double mu = dists[c].Mu[k], sigma = dists[c].Sigma[k]; + int gi = k * nCont + c; + double xstd = (xCont[c][s] - mu) / sigma; + double val = TruncNorm.NormLogPdf(xstd) - logGaussMass[gi] - Math.Log(sigma); + if (aGrid[gi] == bGrid[gi]) + val = double.NaN; + else if (xstd < aGrid[gi] || xstd > bGrid[gi]) + val = double.NegativeInfinity; + sum += val; + } + weighted[s, k] += sum; + } + } + + for (int s = 0; s < nSamples; s++) + { + for (int k = 0; k < nKernels; k++) + weighted[s, k] += Math.Log(mixture.Weights[k]); + } + + var result = new double[nSamples]; + var expBuf = new double[nKernels]; + for (int s = 0; s < nSamples; s++) + { + double max = double.NegativeInfinity; + for (int k = 0; k < nKernels; k++) + max = Math.Max(max, weighted[s, k]); + if (double.IsNegativeInfinity(max)) + max = 0.0; + for (int k = 0; k < nKernels; k++) + expBuf[k] = Math.Exp(weighted[s, k] - max); + result[s] = Math.Log(MathUtil.PairwiseSum(expBuf)) + max; + } + return result; + } + + // ------------------------------------------------------------------ the log Gaussian mass + + /// + /// , now over rented scratch memory, against the replaced + /// allocation-per-call code: the left, right and central cases, NaN bounds, and central counts on both sides of the + /// 2,000-element switch to the vectorized erf. + /// + /// The number of intervals. + [TestMethod] + [DataRow(1)] + [DataRow(17)] + [DataRow(1999)] + [DataRow(2000)] + [DataRow(2001)] + [DataRow(6000)] + public void LogGaussMass_OverRentedScratch_MatchesTheReplacedCodeToTheBit(int n) + { + var rng = new Random(n); + var a = new double[n]; + var b = new double[n]; + for (int i = 0; i < n; i++) + { + // Mostly central intervals, so the central count crosses the switch with n; some left, some right. + double u = rng.NextDouble(); + double lo = u < 0.8 ? -rng.NextDouble() * 4 : (u < 0.9 ? -8 - rng.NextDouble() * 20 : rng.NextDouble() * 6); + a[i] = lo; + b[i] = lo + rng.NextDouble() * 5; + } + if (n > 3) + { + a[1] = double.NaN; + b[2] = double.NaN; + } + + double[] actual = TruncNorm.LogGaussMass(a, b); + double[] expected = ReferenceLogGaussMass(a, b); + for (int i = 0; i < n; i++) + { + if (double.IsNaN(expected[i])) + double.IsNaN(actual[i]).Should().BeTrue($"interval {i} was NaN before"); + else + BitConverter.DoubleToInt64Bits(actual[i]).Should().Be(BitConverter.DoubleToInt64Bits(expected[i]), $"interval {i}"); + } + } + + /// + /// The span overload of picks its path from the span's length, not from + /// the buffer behind it: 1,999 values inside a 5,000-element buffer take the exact path, as a 1,999-element array + /// does. A caller that passed a rented buffer whole would switch paths and change values; this pins the contract. + /// + [TestMethod] + public void ErfComputeArray_OverASpan_ChoosesThePathByTheSpansLength() + { + var rng = new Random(7); + var buffer = new double[5000]; + for (int i = 0; i < buffer.Length; i++) + buffer[i] = (rng.NextDouble() - 0.5) * 8; + + var exact = new double[1999]; + Array.Copy(buffer, exact, 1999); + double[] fromArray = Erf.ComputeArray(exact); + var fromSpan = new double[1999]; + Erf.ComputeArray(new ReadOnlySpan(buffer, 0, 1999), fromSpan); + + for (int i = 0; i < 1999; i++) + { + BitConverter.DoubleToInt64Bits(fromSpan[i]).Should().Be(BitConverter.DoubleToInt64Bits(fromArray[i])); + BitConverter.DoubleToInt64Bits(fromSpan[i]).Should().Be(BitConverter.DoubleToInt64Bits(Erf.Compute(buffer[i])), + "below 2,000 elements every value is the exact erf"); + } + } + + /// + /// The log Gaussian mass as it was computed before item S6: the central intervals gathered into new arrays and + /// passed through the normal CDF, the tails through the scalar log CDF. + /// + /// The standardized lower bounds. + /// The standardized upper bounds. + /// One log mass per interval. + private static double[] ReferenceLogGaussMass(double[] a, double[] b) + { + int n = a.Length; + var result = new double[n]; + var central = new List(); + for (int i = 0; i < n; i++) + { + if (!(a[i] > 0.0) && !(b[i] <= 0.0)) + central.Add(i); + } + if (central.Count > 0) + { + double[] ndtrA = ReferenceNdtr(central.Select(i => a[i]).ToArray()); + double[] ndtrNegB = ReferenceNdtr(central.Select(i => -b[i]).ToArray()); + for (int k = 0; k < central.Count; k++) + result[central[k]] = MathUtil.Log1p(-ndtrA[k] - ndtrNegB[k]); + } + for (int i = 0; i < n; i++) + { + if (a[i] > 0.0) + result[i] = LogDiff(LogNdtr(-a[i]), LogNdtr(-b[i])); + else if (b[i] <= 0.0) + result[i] = LogDiff(LogNdtr(b[i]), LogNdtr(a[i])); + } + return result; + } + + /// The normal CDF as before: 0.5 + 0.5 * erf(a / sqrt(2)) over a new array. + /// The arguments. + /// One probability per argument. + private static double[] ReferenceNdtr(double[] a) + { + double sqrt2 = Math.Sqrt(2.0); + double[] erf = Erf.ComputeArray(a.Select(v => v / sqrt2).ToArray()); + return erf.Select(e => 0.5 + 0.5 * e).ToArray(); + } + + /// The scalar log normal CDF Optuna's _log_ndtr_single computes. + /// The argument. + /// Its log CDF. + private static double LogNdtr(double v) => TruncNorm.LogNdtr(new[] { v })[0]; + + /// Optuna's _log_diff(log_p, log_q) = log_p + log1p(-exp(log_q - log_p)). + /// The larger log term. + /// The log term to subtract. + /// The log of the difference. + private static double LogDiff(double logP, double logQ) => logP + MathUtil.Log1p(-Math.Exp(logQ - logP)); + + // ------------------------------------------------------------------ the stable argsort + + /// + /// is exactly the permutation the replaced LINQ sort returned, over + /// arrays full of ties, NaNs, both zeros and both infinities, and the empty and one-element arrays. + /// + [TestMethod] + public void StableArgsort_IsTheStableOrderByPermutation_WithTiesNaNsAndSignedZeros() + { + var rng = new Random(13); + double[] specials = { double.NaN, 0.0, -0.0, double.PositiveInfinity, double.NegativeInfinity, 1.5, -1.5 }; + for (int round = 0; round < 300; round++) + { + int n = round % 50; + var values = new double[n]; + for (int i = 0; i < n; i++) + { + double u = rng.NextDouble(); + values[i] = u < 0.3 ? specials[rng.Next(specials.Length)] : Math.Round(rng.NextDouble() * 6) - 3; + } + double[] before = (double[])values.Clone(); + + int[] expected = Enumerable.Range(0, n).OrderBy(i => values[i]).ToArray(); + ParzenEstimator.StableArgsort(values).Should().Equal(expected, $"round {round}: [{string.Join(", ", values)}]"); + values.Select(BitConverter.DoubleToInt64Bits).Should().Equal(before.Select(BitConverter.DoubleToInt64Bits), + "the input is not reordered"); + } + } + + // ------------------------------------------------------------------ inputs + + /// A mixture of continuous dimensions with random kernels and weights. + /// The generator. + /// Kernels per dimension, prior included. + /// Continuous dimensions. + /// Every this-many-th dimension is a log dimension. + /// The mixture. + private static MixtureOfProductDistribution RandomMixture(Random rng, int nKernels, int nCont, int logEvery) + { + var dists = new List(); + for (int c = 0; c < nCont; c++) + dists.Add(RandomDimension(rng, nKernels, log: c % logEvery == logEvery - 1)); + return new MixtureOfProductDistribution(RandomWeights(rng, nKernels), dists); + } + + /// One continuous dimension over [0.5, 3.5] (plain, or log so its kernels live in log space). + /// The generator. + /// Kernels, prior included. + /// Whether the dimension is logarithmic. + /// The dimension's kernels. + private static BatchedNumericalDistributions RandomDimension(Random rng, int nKernels, bool log) + { + double low = 0.5, high = 3.5; + double lo = log ? Math.Log(low) : low, hi = log ? Math.Log(high) : high; + var mu = new double[nKernels]; + var sigma = new double[nKernels]; + for (int k = 0; k < nKernels; k++) + { + mu[k] = lo + rng.NextDouble() * (hi - lo); + sigma[k] = (hi - lo) * (0.01 + rng.NextDouble()); + } + return log + ? new BatchedTruncLogNormDistributions(mu, sigma, low, high) + : new BatchedTruncNormDistributions(mu, sigma, low, high); + } + + /// Positive weights summing to one. + /// The generator. + /// How many. + /// The weights. + private static double[] RandomWeights(Random rng, int n) + { + var w = Enumerable.Range(0, n).Select(_ => 0.05 + rng.NextDouble()).ToArray(); + double sum = w.Sum(); + return w.Select(v => v / sum).ToArray(); + } + + /// Candidates in [0.5, 3.5], with a share outside it (their density is zero, -inf in log). + /// The generator. + /// Candidates. + /// Dimensions. + /// The share of coordinates placed outside the bounds. + /// The candidates, one row each. + private static double[,] RandomCandidates(Random rng, int nSamples, int nCont, double outsideShare) + { + var x = new double[nSamples, nCont]; + for (int s = 0; s < nSamples; s++) + { + for (int c = 0; c < nCont; c++) + { + double u = rng.NextDouble(); + x[s, c] = u < outsideShare ? (rng.Next(2) == 0 ? 0.4 : 3.6) : 0.5 + rng.NextDouble() * 3.0; + } + } + return x; + } + + /// An array of copies of . + /// The length. + /// The value. + /// The array. + private static double[] Fill(int n, double value) => Enumerable.Repeat(value, n).ToArray(); +} diff --git a/test/OptunaSharp.Tests/Samplers/TpeSplitCacheTests.cs b/test/OptunaSharp.Tests/Samplers/TpeSplitCacheTests.cs new file mode 100644 index 0000000..c7b7a04 --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/TpeSplitCacheTests.cs @@ -0,0 +1,223 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Distributions; +using OptunaSharp.Internal; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S5: TPE reuses its last below/above split while the +/// same trial records are split again, which independent TPE does once per parameter; and item S3 for TPE: the +/// constant-liar history read skips the trial being sampled without reading its record. The cache returns exactly +/// what returns, and it is reused only for the very same records of the +/// same study with the same size of the preferred group. +/// +[TestClass] +public class TpeSplitCacheTests +{ + /// + /// Two-objective histories with completed, pruned, failed-constraint and running trials: the cache's split equals + /// for every size of the preferred group, and splitting the same records + /// again reuses it. + /// + /// Seeds the history. + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void TheCachedSplit_IsSplitTrials_AndTheSameRecordsReuseIt(int seed) + { + Study study = MultiObjectiveHistory(seed, out List trials); + var cache = new TpeSplitCache(); + int computed = 0; + for (int nBelow = 0; nBelow <= trials.Count(t => t.State != TrialState.Running); nBelow++) + { + var (below, above) = cache.Split(study, trials, nBelow); + cache.Computed.Should().Be(++computed, "a new size of the preferred group is a new split"); + var (expectedBelow, expectedAbove) = TpeTrialSplit.SplitTrials(study, trials, nBelow); + below.Select(t => t.Number).Should().Equal(expectedBelow.Select(t => t.Number), $"below, n_below={nBelow}"); + above.Select(t => t.Number).Should().Equal(expectedAbove.Select(t => t.Number), $"above, n_below={nBelow}"); + + var (againBelow, againAbove) = cache.Split(study, trials.ToList(), nBelow); + cache.Computed.Should().Be(computed, "the same records, in a new list, reuse the split"); + againBelow.Select(t => t.Number).Should().Equal(below.Select(t => t.Number)); + againAbove.Select(t => t.Number).Should().Equal(above.Select(t => t.Number)); + } + } + + /// + /// Anything but the very same records misses: one record more or less, the records in another order, a running + /// trial whose record was replaced by a write after it was read, the same history shape in another study. + /// + [TestMethod] + public void AnythingButTheSameRecords_SplitsAfresh() + { + Study study = MultiObjectiveHistory(4, out List trials); + var cache = new TpeSplitCache(); + int nBelow = 3; + cache.Split(study, trials, nBelow); + cache.Computed.Should().Be(1); + + cache.Split(study, trials.Take(trials.Count - 1).ToList(), nBelow); + cache.Computed.Should().Be(2, "one record fewer"); + cache.Split(study, Enumerable.Reverse(trials).ToList(), nBelow); + cache.Computed.Should().Be(3, "another order"); + cache.Split(study, trials, nBelow); + cache.Computed.Should().Be(4, "back to the original records: the cache holds the last split only"); + + // A running trial written after it was read gets a new record (in-memory item S4 copies a handed-out record). + FrozenTrial running = trials.First(t => t.State == TrialState.Running); + study.Storage.SetTrialUserAttr(running.TrialId, "written", 1L); + List reread = study.GetTrials(deepcopy: false, + states: new[] { TrialState.Complete, TrialState.Pruned, TrialState.Running }); + reread.Should().NotContain(t => ReferenceEquals(t, running)); + cache.Split(study, reread, nBelow); + cache.Computed.Should().Be(5, "the running trial's record changed"); + + Study other = MultiObjectiveHistory(4, out List otherTrials); + cache.Split(other, otherTrials, nBelow); + cache.Computed.Should().Be(6, "another study's records, even with the same values"); + } + + /// The lists the cache returns are the caller's: changing them leaves the next reuse intact. + [TestMethod] + public void TheReturnedLists_AreCopies() + { + Study study = MultiObjectiveHistory(5, out List trials); + var cache = new TpeSplitCache(); + var (below, above) = cache.Split(study, trials, 4); + int[] belowNumbers = below.Select(t => t.Number).ToArray(); + int[] aboveNumbers = above.Select(t => t.Number).ToArray(); + below.Clear(); + above.Reverse(); + + var (againBelow, againAbove) = cache.Split(study, trials, 4); + cache.Computed.Should().Be(1); + againBelow.Select(t => t.Number).Should().Equal(belowNumbers); + againAbove.Select(t => t.Number).Should().Equal(aboveNumbers); + } + + /// Null arguments are refused before anything is cached. + [TestMethod] + public void NullArguments_AreRefused() + { + Study study = MultiObjectiveHistory(6, out List trials); + var cache = new TpeSplitCache(); + Action noStudy = () => cache.Split(null!, trials, 1); + Action noTrials = () => cache.Split(study, null!, 1); + noStudy.Should().Throw(); + noTrials.Should().Throw(); + cache.Computed.Should().Be(0); + } + + /// + /// A multi-objective study sampled by independent TPE splits its history once per trial, not once per parameter: + /// every parameter of a trial reads the same records. Constant liar is on (Optuna 5's default), so this also shows + /// the read leaves the trial being sampled out without touching its record: a read of it would change its record + /// at the next suggestion and make every parameter split afresh. + /// + [TestMethod] + public void IndependentMultiObjectiveTpe_SplitsOncePerTrial_NotOncePerParameter() + { + var sampler = new TPESampler(seed: 3, nStartupTrials: 5); + var storage = new InMemoryStorage(); + Study study = Optuna.CreateStudy(storage: storage, sampler: sampler, + directions: new[] { StudyDirection.Minimize, StudyDirection.Minimize }); + const int trials = 25, parameters = 8; + study.Optimize(t => + { + double a = 0, b = 0; + for (int i = 0; i < parameters; i++) + { + double v = t.SuggestFloat($"x{i}", -5, 5); + a += v * v; + b += (v - 1) * (v - 1); + } + return new[] { a, b }; + }, nTrials: trials); + + sampler.SplitCache.Computed.Should().Be(trials - 5, "one split per trial after the startup trials"); + study.Trials.Should().HaveCount(trials).And.OnlyContain(t => t.State == TrialState.Complete); + } + + /// + /// With another worker's trial running, constant liar still sees it, and still leaves out the trial being sampled: + /// the history the sampler splits is every finished trial plus the other running one. + /// + [TestMethod] + public void ConstantLiar_SeesOtherRunningTrials_AndLeavesOutTheCurrentOne() + { + var sampler = new TPESampler(seed: 4, nStartupTrials: 3); + Study study = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: sampler, + directions: new[] { StudyDirection.Minimize, StudyDirection.Minimize }); + study.Optimize(t => new[] { t.SuggestFloat("x", 0, 1), t.SuggestFloat("y", 0, 1) }, nTrials: 6); + + Trial otherWorker = study.Ask(); + otherWorker.SuggestFloat("x", 0, 1); + Trial current = study.Ask(); + int before = sampler.SplitCache.Computed; + current.SuggestFloat("x", 0, 1); + current.SuggestFloat("y", 0, 1); + + sampler.SplitCache.Computed.Should().Be(before + 1, "both parameters of the current trial split the same records"); + // The running trial the other worker holds is in the history; the current trial is not: 6 finished + 1 running. + var (below, above) = TpeTrialSplit.SplitTrials(study, + study.GetTrials(deepcopy: false, states: new[] { TrialState.Complete, TrialState.Pruned, TrialState.Running }) + .Where(t => t.Number != current.Number).ToList(), TpeTrialSplit.DefaultGammaMultiObjective(6)); + (below.Count + above.Count).Should().Be(7); + above.Should().Contain(t => t.Number == otherWorker.Number); + study.Tell(otherWorker, new[] { 0.5, 0.5 }); + study.Tell(current, new[] { 0.5, 0.5 }); + } + + /// A worker instance for gets a split cache of its own. + [TestMethod] + public void AWorkerInstance_HasItsOwnSplitCache() + { + var sampler = new TPESampler(seed: 1); + var worker = (TPESampler)sampler.CreateWorkerInstance()!; + worker.SplitCache.Should().NotBeSameAs(sampler.SplitCache); + } + + /// + /// A two-objective study on an in-memory storage with 14 finished trials (completed, pruned with and without + /// intermediate values, completed but infeasible) and 2 running ones, and the records TPE's constant-liar read + /// returns for it. + /// + /// Seeds the values. + /// The study's completed, pruned and running records, by trial number. + /// The study. + private static Study MultiObjectiveHistory(int seed, out List trials) + { + var rng = new Random(seed); + Study study = Optuna.CreateStudy(storage: new InMemoryStorage(), + directions: new[] { StudyDirection.Minimize, StudyDirection.Maximize }); + var storage = study.Storage; + for (int n = 0; n < 16; n++) + { + int id = storage.CreateNewTrial(study.StudyId); + storage.SetTrialParam(id, "x", rng.NextDouble(), new FloatDistribution(0, 1)); + if (n % 8 == 7) + continue; // stays running + if (n % 5 == 4) + { + if (n % 2 == 0) + storage.SetTrialIntermediateValue(id, n, rng.NextDouble()); + storage.SetTrialStateValues(id, TrialState.Pruned); + continue; + } + if (n % 6 == 5) + storage.SetTrialSystemAttr(id, "constraints", new List { 0.5 + rng.NextDouble() }); + // Values on a small grid, so ranks and ties are plentiful. + storage.SetTrialStateValues(id, TrialState.Complete, + new[] { Math.Round(rng.NextDouble() * 4) / 4, Math.Round(rng.NextDouble() * 4) / 4 }); + } + trials = study.GetTrials(deepcopy: false, + states: new[] { TrialState.Complete, TrialState.Pruned, TrialState.Running }); + return study; + } +} diff --git a/test/OptunaSharp.Tests/Samplers/ZeroCopyHistoryReadTests.cs b/test/OptunaSharp.Tests/Samplers/ZeroCopyHistoryReadTests.cs new file mode 100644 index 0000000..c1fd305 --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/ZeroCopyHistoryReadTests.cs @@ -0,0 +1,151 @@ +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Collections; +using OptunaSharp.Distributions; +using OptunaSharp.Samplers; +using OptunaSharp.SearchSpace; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S3 for the QMC sampler and the group-decomposed +/// search space: both now scan the history through Study.GetTrialsForSampler (the zero-copy view in memory) +/// instead of a copied, filtered list, and QMC stops at the first match instead of sorting. Each test builds a history +/// with trials finished out of order and in every state, and compares the result with Optuna's read, done the way +/// Optuna does it: copy, filter by state, sort by number. +/// +[TestClass] +public class ZeroCopyHistoryReadTests +{ + /// + /// QMC's relative search space is the distributions of the lowest-numbered COMPLETE or PRUNED trial, whatever + /// order the trials finished in and whatever states lie before it. + /// + /// The state the lowest-numbered finished trial ends in. + [TestMethod] + [DataRow(TrialState.Complete)] + [DataRow(TrialState.Pruned)] + public void Qmc_TheSearchSpaceComesFromTheLowestNumberedFinishedTrial(TrialState firstFinishedState) + { + var sampler = new QMCSampler(seed: 1, warnIndependentSampling: false); + Study study = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: sampler); + BaseStorage storage = study.Storage; + + int running = storage.CreateNewTrial(study.StudyId); // number 0, never finishes + storage.SetTrialParam(running, "a", 0.5, new FloatDistribution(0, 1)); + int failed = storage.CreateNewTrial(study.StudyId); // number 1, fails + storage.SetTrialParam(failed, "b", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialStateValues(failed, TrialState.Fail); + int late = storage.CreateNewTrial(study.StudyId); // number 2, finishes after number 3 + storage.SetTrialParam(late, "x", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialParam(late, "y", 2, new IntDistribution(0, 5)); + int early = storage.CreateNewTrial(study.StudyId); // number 3, finishes first + storage.SetTrialParam(early, "z", 0.5, new FloatDistribution(0, 2)); + storage.SetTrialStateValues(early, TrialState.Complete, new[] { 1.0 }); + storage.SetTrialStateValues(late, firstFinishedState, firstFinishedState == TrialState.Complete ? new[] { 2.0 } : null); + + FrozenTrial current = storage.GetTrial(storage.CreateNewTrial(study.StudyId)); + OrderedDict space = sampler.InferRelativeSearchSpace(study, current); + + // Optuna: sorted(study._get_trials(deepcopy=False, states=(COMPLETE, PRUNED)), key=number)[0].distributions. + FrozenTrial expected = study.GetTrials(deepcopy: false, states: new[] { TrialState.Complete, TrialState.Pruned }) + .OrderBy(t => t.Number).First(); + expected.TrialId.Should().Be(late); + space.Keys.Should().Equal(expected.Distributions.Keys); + foreach (var (name, distribution) in expected.Distributions) + space[name].Should().BeSameAs(distribution); + } + + /// + /// Before any trial finished, QMC still takes the union of the running trials' distributions, as Optuna does for + /// the initial trial, and an empty space when nothing is running either. + /// + [TestMethod] + public void Qmc_BeforeAnyTrialFinished_UsesTheRunningTrials() + { + // Two studies: the running-trial read uses the per-thread cache, which lives until the next Ask, so one study + // per question keeps each answer about the history built for it. + var empty = new QMCSampler(seed: 2, warnIndependentSampling: false); + Study bare = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: empty); + FrozenTrial nothing = bare.Storage.GetTrial(bare.Storage.CreateNewTrial(bare.StudyId)); + empty.InferRelativeSearchSpace(bare, nothing).Should().BeEmpty(); + + var sampler = new QMCSampler(seed: 2, warnIndependentSampling: false); + Study study = Optuna.CreateStudy(storage: new InMemoryStorage(), sampler: sampler); + BaseStorage storage = study.Storage; + int first = storage.CreateNewTrial(study.StudyId); + storage.SetTrialParam(first, "x", 0.5, new FloatDistribution(0, 1)); + int other = storage.CreateNewTrial(study.StudyId); + storage.SetTrialParam(other, "x", 0.5, new FloatDistribution(0, 1)); + storage.SetTrialParam(other, "y", 0.5, new FloatDistribution(0, 1)); + FrozenTrial current = storage.GetTrial(storage.CreateNewTrial(study.StudyId)); + sampler.InferRelativeSearchSpace(study, current).Keys.Should().Equal("x", "y"); + } + + /// + /// The group-decomposed search space over a history with every state, parameters that come and go, and trials + /// finished out of order equals Optuna's decomposition of the copied, filtered history: the same groups, in the + /// same order, holding the same distribution objects, with and without pruned trials and with and without the + /// per-trial cache. + /// + /// Whether pruned trials count. + /// Whether the read may use the per-thread cache. + [TestMethod] + [DataRow(false, false)] + [DataRow(false, true)] + [DataRow(true, false)] + [DataRow(true, true)] + public void GroupDecomposed_EqualsOptunasDecompositionOfTheFilteredHistory(bool includePruned, bool useCache) + { + Study study = Optuna.CreateStudy(storage: new InMemoryStorage()); + BaseStorage storage = study.Storage; + string[][] paramSets = + { + new[] { "a", "b" }, new[] { "a", "c" }, new[] { "d" }, new[] { "a", "b", "e" }, new[] { "c" }, + new[] { "f", "a" }, new[] { "b" }, new[] { "e", "d" }, + }; + TrialState[] finalStates = + { + TrialState.Complete, TrialState.Pruned, TrialState.Fail, TrialState.Complete, TrialState.Running, + TrialState.Pruned, TrialState.Complete, TrialState.Complete, + }; + var ids = new List(); + for (int n = 0; n < paramSets.Length; n++) + { + int id = storage.CreateNewTrial(study.StudyId); + ids.Add(id); + foreach (string name in paramSets[n]) + storage.SetTrialParam(id, name, 0.5, new FloatDistribution(0, 1)); + } + // Finish in reverse order, so the finishing order disagrees with the trial numbers. + for (int n = paramSets.Length - 1; n >= 0; n--) + { + if (finalStates[n] == TrialState.Running) + continue; + storage.SetTrialStateValues(ids[n], finalStates[n], finalStates[n] == TrialState.Complete ? new[] { 1.0 } : null); + } + + var space = new GroupDecomposedSearchSpace(includePruned); + SearchSpaceGroup actual = space.Calculate(study, useCache); + + var expected = new SearchSpaceGroup(); + TrialState[] states = includePruned ? new[] { TrialState.Complete, TrialState.Pruned } : new[] { TrialState.Complete }; + foreach (FrozenTrial trial in study.GetTrials(deepcopy: false, states: states)) + expected.AddDistributions(trial.DistributionsDict); + + actual.SearchSpaces.Should().HaveCount(expected.SearchSpaces.Count); + for (int g = 0; g < expected.SearchSpaces.Count; g++) + { + actual.SearchSpaces[g].Keys.Should().Equal(expected.SearchSpaces[g].Keys, $"group {g}"); + foreach (var (name, distribution) in expected.SearchSpaces[g]) + actual.SearchSpaces[g][name].Should().BeSameAs(distribution); + } + + // A second call over the same history is the same decomposition, as Optuna's re-adding every trial is. + SearchSpaceGroup again = space.Calculate(study, useCache); + again.SearchSpaces.Select(s => string.Join(",", s.Keys)) + .Should().Equal(expected.SearchSpaces.Select(s => string.Join(",", s.Keys))); + } +} From 9043b43cb6a438a9d37f58fe101674314f42f576 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 00:39:00 +0200 Subject: [PATCH 012/145] perf(storage): CachedStorage reads its cache in number order and wraps it as a view; database loads parse each distribution once Plan 13 (docs/plans/13-rustuna-derived-performance.md), phase 4: item S10, and item S9 as its measurement turned out. Same records, same order, same values as before. S10, CachedStorage (the wrapper Optuna puts around every RDB storage): - GetAllTrials sorted the cached dictionary by trial number on every read, and GetTrialsPrefixView fell back to that copy, so the genetic samplers, which take the view on every ask, paid O(n log n) per ask on a database storage. - StudyInfo.ByNumber mirrors the cache by trial number (AddTrialsToCache keeps it in step). A number not cached yet holds a null placeholder counted in MissingNumbers: a local create can cache trial n + 1 before a sync brings in a trial n another worker created. - With no placeholder, GetAllTrials walks the mirror (the same records in the same order as the sort) and GetTrialsPrefixView wraps it in a TrialsPrefixView after the same synchronization GetAllTrials does. Cached records are never changed in place (writes go to the backend; a sync swaps in fresh records), so the view needs no lock. With a placeholder, both use the sorted copy, as before. - HasZeroCopyTrialsView stays false here: samplers that share the per-trial cached list (GetTrialsForSampler) keep it, since a view read means a synchronization, which is a database round trip. S9, measured first as the plan asked (10,000-trial, 40-parameter CMA-ES study on SQLite, cold load with a new RDBStorage, i9-13900K, Release): - GetAllTrials took 2.1 s and allocated 1.22 GB. Decoding every attribute value (26k values, 30 MB of JSON, the CMA-ES pickles included) took 0.06-0.08 s of it: lazy attribute decoding would save under 4 % and move malformed-JSON errors from load to access. Not built. - The profile showed where the JSON time really goes: Distribution.FromJson for every parameter row, 400,000 parses of 40 distinct texts, about 40 % of the load. - Both RDB storages (Storage.Sqlite and Storage.EFCore) now parse each distinct distribution text once per load through the journal storage's JournalDistributionCache (plan 10, item 2.7): the shared object equals a fresh parse, only the three current distribution types are shared (the deprecated ones are built per use, so their warnings fire as before), and the cache is local to the call. Cold load: 2.1 s -> 1.4-1.5 s, 1.22 GB -> 479 MB. Tests: - CachedStorageOrderedReadsTests (unit, over a fake database: an in-memory store returning fresh records per read that can hide another worker's trial from a sync): every state filter with and without deep copies returns the backend's trials in number order over a history finished out of order; the view is a TrialsPrefixView holding the very records of the uncopied read and sees an unfinished trial a later sync refreshed; a gap falls back to the sorted copy (the view to a List) until a sync fills it; seeded NSGA-II (which reads the view every ask) and CMA-ES studies on the cached fake database equal the in-memory runs trial by trial. - SqliteRdbStorageTests.Reload_ParsesEachDistributionTextOnce_SharingOnlyTheCurrentTypes: a reloaded study shares one object per distinct text, each equal to a fresh parse of its stored text; a -0.0 bound is its own object and prints -0.0; a deprecated UniformDistribution is never shared; parameter values unchanged. Results: the CI unit filter 2056/2056 on net8.0 and on net10.0; the storage suite on net10.0 452 passed, 144 not run here (Redis, PostgreSQL and MySQL services), 0 failed; live-Python interop for the SQLite, EF Core, storage-URL and storage-recipe cases 33/33 (Python writes and reads databases this loader parses); documentation gate 0 issues. --- .../Storages/CachedStorage.cs | 87 +++++++- .../Storages/Rdb/RDBStorage.cs | 7 +- .../Storages/Rdb/RDBStorage.cs | 17 +- .../SqliteRdbStorageTests.cs | 49 +++++ .../CachedStorageOrderedReadsTests.cs | 206 ++++++++++++++++++ 5 files changed, 357 insertions(+), 9 deletions(-) create mode 100644 test/OptunaSharp.Tests/Storages/CachedStorageOrderedReadsTests.cs diff --git a/src/OptunaSharp.Core/Storages/CachedStorage.cs b/src/OptunaSharp.Core/Storages/CachedStorage.cs index 020ed64..0b728b7 100644 --- a/src/OptunaSharp.Core/Storages/CachedStorage.cs +++ b/src/OptunaSharp.Core/Storages/CachedStorage.cs @@ -85,6 +85,22 @@ private sealed class StudyInfo // Trial number to corresponding FrozenTrial. /// Cached trials keyed by their per-study number. internal readonly Dictionary Trials = new(); + + /// + /// The same cached records indexed by trial number (Rustuna plan 13, item S10), so reads walk them in number + /// order instead of sorting the dictionary on every call. A number not cached yet holds a null placeholder, + /// counted in : a local create can cache trial n + 1 before a sync brings in a + /// trial n another worker created. + /// + /// + /// Elements are only ever replaced by whole records the backend returned, never changed in place, and + /// placeholders are only appended or filled, never written over a record. That is what lets + /// wrap this list without the lock. + /// + internal readonly List ByNumber = new(); + + /// How many null placeholders holds; zero means it is the whole cache in order. + internal int MissingNumbers; // Trials which require storage access to read latest attributes. /// Trial identifiers that must be refreshed because their state can still change. internal readonly HashSet UnfinishedTrialIds = new(); @@ -368,15 +384,58 @@ public override List GetAllTrials( { var study = _studies[studyId]; // We need to sort trials by their number because some samplers assume this behavior. - - IEnumerable trials = states is not null - ? study.Trials.Values.Where(t => states.Contains(t.State)) - : study.Trials.Values; - var sorted = trials.OrderBy(t => t.Number).ToList(); + List sorted; + if (study.MissingNumbers == 0) + { + // The number-ordered mirror is the whole cache in order (plan 13, item S10): the same records in the same + // order as the sort below, without sorting on every read. + sorted = new List(study.ByNumber.Count); + foreach (var trial in study.ByNumber) + { + if (states is null || states.Contains(trial.State)) + sorted.Add(trial); + } + } + else + { + IEnumerable trials = states is not null + ? study.Trials.Values.Where(t => states.Contains(t.State)) + : study.Trials.Values; + sorted = trials.OrderBy(t => t.Number).ToList(); + } return deepcopy ? sorted.Select(t => t.DeepCopy()).ToList() : sorted; } } + /// Storage identifier of the study; distinct from its name. + /// + /// + /// + /// Synchronizes with the backend exactly as does, then wraps the number-ordered + /// cache in a instead of sorting and copying it (Rustuna plan 13, item S10): + /// the genetic samplers take this view on every ask, which made each ask O(n log n) on a database storage. + /// The view holds the same records GetAllTrials(deepcopy: false) would return, in the same order; + /// a later synchronization can replace an unfinished trial's record with a fresher one, which a read of the + /// view then returns, as the view's contract says. + /// + /// + /// When a trial number is not cached yet (another worker's trial a sync has not brought in), the view could + /// not be indexed by number, so the copying read is used instead, as before. + /// + /// + public override IReadOnlyList GetTrialsPrefixView(int studyId) + { + ReadTrialsFromRemoteStorage(studyId); + + lock (_lock) + { + var study = _studies[studyId]; + if (study.MissingNumbers == 0) + return new TrialsPrefixView(study.ByNumber, study.ByNumber.Count); + } + return GetAllTrials(studyId, deepcopy: false); + } + /// Storage identifier of the study; distinct from its name. /// Refreshes unfinished and newly created trials before advancing the completed-trial high-water mark. private void ReadTrialsFromRemoteStorage(int studyId) @@ -422,6 +481,24 @@ private void AddTrialsToCache(int studyId, List trials) _trialIdToStudyIdAndNumber[trial.TrialId] = (studyId, trial.Number); _studyIdAndNumberToTrialId[(studyId, trial.Number)] = trial.TrialId; study.Trials[trial.Number] = trial; + + // The number-ordered mirror (item S10): placeholders for numbers not cached yet, then this record. + var byNumber = study.ByNumber; + while (byNumber.Count < trial.Number) + { + byNumber.Add(null!); + study.MissingNumbers++; + } + if (byNumber.Count == trial.Number) + { + byNumber.Add(trial); + } + else + { + if (byNumber[trial.Number] is null) + study.MissingNumbers--; + byNumber[trial.Number] = trial; + } } } diff --git a/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs b/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs index 8eb48f3..8f192a7 100644 --- a/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs +++ b/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs @@ -17,6 +17,7 @@ using OptunaSharp.Internal; using OptunaSharp.Logging; using OptunaSharp.Storages; +using OptunaSharp.Storages.Journal; namespace OptunaSharp.Storages.EFCore; @@ -1269,6 +1270,10 @@ private static List GetTrialsInternal( .Select(a => new { a.TrialId, a.Key, a.ValueJson }) .AsEnumerable().GroupBy(a => a.TrialId!.Value).ToDictionary(g => g.Key, g => g.ToList()); + // One parse per distinct distribution text for this load (plan 13, item S9 as measured on SQLite: parsing every + // parameter row was 40 % of a 10,000-trial cold load). The journal storage's cache (plan 10, item 2.7) shares + // the parsed object, which equals a fresh parse; it is local to this call, so no thread ever shares it. + var distributionCache = new JournalDistributionCache(); var result = new List(trialRows.Count); foreach (var row in trialRows) { @@ -1286,7 +1291,7 @@ private static List GetTrialsInternal( { foreach (var p in paramRows) { - var distribution = Distribution.FromJson(p.DistributionJson!); + var distribution = distributionCache.FromJson(p.DistributionJson!); @params[p.ParamName!] = distribution.ToExternalRepr(p.ParamValue!.Value); distributions[p.ParamName!] = distribution; } diff --git a/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs b/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs index 17b5ffb..09d6c97 100644 --- a/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs +++ b/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs @@ -1172,6 +1172,11 @@ private List GetTrialsInternal( var userByTrial = LoadTrialAttrs("trial_user_attributes", "trial_user_attribute_id", trialIds); var systemByTrial = LoadTrialAttrs("trial_system_attributes", "trial_system_attribute_id", trialIds); + // One parse per distinct distribution text for this load (plan 13, item S9 as measured): a 40-parameter study + // repeats 40 texts, and parsing every parameter row was 40 % of a 10,000-trial cold load, where the attribute + // JSON lazy decoding would have deferred was 3-4 %. The journal storage's cache (plan 10, item 2.7) shares the + // parsed object, which equals a fresh parse; it is local to this call, so no thread ever shares it. + var distributionCache = new OptunaSharp.Storages.Journal.JournalDistributionCache(); var result = new List(trialRows.Count); foreach (var row in trialRows) { @@ -1181,7 +1186,8 @@ private List GetTrialsInternal( valuesByTrial.GetValueOrDefault(row.TrialId), intermediateByTrial.GetValueOrDefault(row.TrialId), userByTrial.GetValueOrDefault(row.TrialId), - systemByTrial.GetValueOrDefault(row.TrialId))); + systemByTrial.GetValueOrDefault(row.TrialId), + distributionCache)); } return result; } @@ -1197,7 +1203,11 @@ private List GetTrialsInternal( /// Intermediate-report rows indexed by step, or null when absent. /// Serialized user attributes for the trial, or null when absent. /// Serialized system attributes for the trial, or null when absent. + /// + /// Parses each distinct distribution text once for the whole load; the distributions it returns equal fresh parses. + /// /// A detached snapshot carrying the supplied trial data. + /// A stored distribution text is not a valid distribution. private FrozenTrial BuildFrozenTrial( int trialId, int number, @@ -1208,7 +1218,8 @@ private FrozenTrial BuildFrozenTrial( List<(int Objective, double? Value, string Type)>? valueRows, List<(long Step, double? Value, string Type)>? intermediateRows, List<(string Key, string Json)>? userRows, - List<(string Key, string Json)>? systemRows) + List<(string Key, string Json)>? systemRows, + OptunaSharp.Storages.Journal.JournalDistributionCache distributionCache) { List? values = null; if (valueRows is { Count: > 0 }) @@ -1224,7 +1235,7 @@ private FrozenTrial BuildFrozenTrial( { foreach (var (_, name, value, distJson) in paramRows.OrderBy(p => p.ParamId)) { - var distribution = Distribution.FromJson(distJson); + var distribution = distributionCache.FromJson(distJson); @params[name] = distribution.ToExternalRepr(value); distributions[name] = distribution; } diff --git a/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs b/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs index 1aa7b99..90e3997 100644 --- a/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs +++ b/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs @@ -56,6 +56,55 @@ private string TempDbPath() private static OrderedDict QuadDist() => new() { ["x"] = new FloatDistribution(-10, 10) }; + // ------------------------------------------------------------------ plan 13: one parse per distribution text + + /// + /// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S9 as measured: a load parses each distinct + /// distribution text once. Reloading a study gives every trial of a parameter the same distribution object, equal + /// to a fresh parse of its stored text; a different text (a -0.0 bound, another range, a categorical) is its own + /// object, and the deprecated types are never shared, so their warnings fire per use as before. + /// + [TestMethod] + public void Reload_ParsesEachDistributionTextOnce_SharingOnlyTheCurrentTypes() + { + string path = TempDbPath(); + string url = $"sqlite:///{path.Replace('\\', '/')}"; + int studyId; + using (var storage = new RDBStorage(url)) + { + studyId = storage.CreateNewStudy(new[] { StudyDirection.Minimize }, "shared-distributions"); + for (int n = 0; n < 6; n++) + { + int id = storage.CreateNewTrial(studyId); + storage.SetTrialParam(id, "x", 0.25 * n, new FloatDistribution(0, 2)); + storage.SetTrialParam(id, "c", n % 2, new CategoricalDistribution(new CategoricalChoice[] { "a", "b" })); + storage.SetTrialParam(id, "z", 0.5, n < 3 ? new FloatDistribution(0.0, 1) : new FloatDistribution(-0.0, 1)); +#pragma warning disable CS0618 // the deprecated family is the point of this case + storage.SetTrialParam(id, "u", 0.5, new UniformDistribution(0, 1)); +#pragma warning restore CS0618 + storage.SetTrialStateValues(id, TrialState.Complete, new[] { (double)n }); + } + } + + using var reopened = new RDBStorage(url); + List trials = reopened.GetAllTrials(studyId, deepcopy: false); + trials.Should().HaveCount(6); + foreach (FrozenTrial trial in trials) + { + trial.Distributions["x"].Should().BeSameAs(trials[0].Distributions["x"]); + trial.Distributions["c"].Should().BeSameAs(trials[0].Distributions["c"]); + foreach (var (name, distribution) in trial.Distributions) + distribution.Should().Be(Distribution.FromJson(Distribution.ToJson(distribution)), name); + } + trials[0].Distributions["z"].Should().BeSameAs(trials[2].Distributions["z"]); + trials[3].Distributions["z"].Should().BeSameAs(trials[5].Distributions["z"]); + trials[0].Distributions["z"].Should().NotBeSameAs(trials[3].Distributions["z"], "-0.0 is stored as its own text"); + trials[3].Distributions["z"].ToString().Should().Contain("low=-0.0"); + trials[0].Distributions["u"].Should().NotBeSameAs(trials[1].Distributions["u"], "deprecated types are never shared"); + trials.Select(t => t.Params["x"]).Should().Equal(0.0, 0.25, 0.5, 0.75, 1.0, 1.25); + trials.Select(t => t.Params["c"]).Should().Equal("a", "b", "a", "b", "a", "b"); + } + // ------------------------------------------------------------------ schema / version pins [TestMethod] diff --git a/test/OptunaSharp.Tests/Storages/CachedStorageOrderedReadsTests.cs b/test/OptunaSharp.Tests/Storages/CachedStorageOrderedReadsTests.cs new file mode 100644 index 0000000..ad75b35 --- /dev/null +++ b/test/OptunaSharp.Tests/Storages/CachedStorageOrderedReadsTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Distributions; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Storages; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S10: keeps its cached +/// trials in number order as well, so no longer sorts the cache on every read +/// and wraps it instead of copying. The backend here is a fake +/// database: an in-memory store that returns fresh records on every read, as a database does, and can hide trials +/// from a synchronization, as another worker's trial is hidden until a sync sees it. +/// +[TestClass] +public class CachedStorageOrderedReadsTests +{ + private static readonly StudyDirection[] Minimize = { StudyDirection.Minimize }; + + /// + /// Over a history finished out of order and in every state, every read returns the trials the backend holds, in + /// number order, with and without each state filter, and with deep copies. + /// + [TestMethod] + public void GetAllTrials_ReturnsTheBackendsTrialsInNumberOrder_ForEveryFilter() + { + var backend = new FakeDatabase(); + var cached = new CachedStorage(backend); + int studyId = cached.CreateNewStudy(Minimize, "ordered"); + var ids = new List(); + for (int n = 0; n < 30; n++) + { + int id = cached.CreateNewTrial(studyId); + ids.Add(id); + cached.SetTrialParam(id, "x", n / 30.0, new FloatDistribution(0, 1)); + if (n % 3 == 0) + _ = cached.GetAllTrials(studyId, deepcopy: false); // syncs land between the creates + } + TrialState[] finals = { TrialState.Complete, TrialState.Pruned, TrialState.Fail, TrialState.Running }; + for (int n = 29; n >= 0; n--) + { + TrialState state = finals[n % finals.Length]; + if (state != TrialState.Running) + cached.SetTrialStateValues(ids[n], state, state == TrialState.Complete ? new[] { (double)n } : null); + } + + foreach (TrialState[]? states in new[] + { + null, new[] { TrialState.Complete }, new[] { TrialState.Complete, TrialState.Pruned }, + new[] { TrialState.Running }, new[] { TrialState.Waiting }, + }) + { + foreach (bool deepcopy in new[] { false, true }) + { + List actual = cached.GetAllTrials(studyId, deepcopy, states); + List expected = backend.GetAllTrials(studyId, deepcopy: true, states); + actual.Select(Describe).Should().Equal(expected.Select(Describe), + $"states [{(states is null ? "all" : string.Join(",", states))}], deepcopy {deepcopy}"); + } + } + } + + /// + /// With the cache contiguous, the view is a zero-copy window holding the very records the uncopied read returns; + /// a later synchronization that refreshes an unfinished trial is seen through it, as the view's contract says. + /// + [TestMethod] + public void TheView_IsZeroCopy_AndSeesARefreshedUnfinishedTrial() + { + var backend = new FakeDatabase(); + var cached = new CachedStorage(backend); + int studyId = cached.CreateNewStudy(Minimize, "view"); + int done = cached.CreateNewTrial(studyId); + cached.SetTrialStateValues(done, TrialState.Complete, new[] { 1.0 }); + int running = cached.CreateNewTrial(studyId); + + IReadOnlyList view = cached.GetTrialsPrefixView(studyId); + view.Should().BeOfType(); + List list = cached.GetAllTrials(studyId, deepcopy: false); + view.Should().HaveCount(2); + view[0].Should().BeSameAs(list[0]); + view[1].Should().BeSameAs(list[1]); + + // Another worker writes the running trial; the next synchronization brings the fresher record in. + backend.SetTrialParam(running, "y", 0.5, new FloatDistribution(0, 1)); + _ = cached.GetAllTrials(studyId, deepcopy: false); + view[1].Params.Should().ContainKey("y"); + view[0].Should().BeSameAs(list[0], "a finished record is never replaced"); + } + + /// + /// A trial number not cached yet (another worker's trial a sync has not seen, while this process cached a later + /// one) leaves a gap: reads fall back to sorting the cache and the view to a copied list, both still in number + /// order; once a sync brings the missing trial in, the view is zero-copy again and holds every trial. + /// + [TestMethod] + public void AGap_FallsBackToTheSortedCopy_UntilASyncFillsIt() + { + var backend = new FakeDatabase(); + var cached = new CachedStorage(backend); + int studyId = cached.CreateNewStudy(Minimize, "gap"); + cached.CreateNewTrial(studyId); // number 0 + cached.CreateNewTrial(studyId); // number 1 + int other = backend.CreateNewTrial(studyId); // number 2, another worker's, not seen yet + backend.Hidden.Add(other); + cached.CreateNewTrial(studyId); // number 3, cached locally beside a gap at 2 + + cached.GetAllTrials(studyId, deepcopy: false).Select(t => t.Number).Should().Equal(0, 1, 3); + IReadOnlyList fallback = cached.GetTrialsPrefixView(studyId); + fallback.Should().NotBeOfType(); + fallback.Select(t => t.Number).Should().Equal(0, 1, 3); + + backend.Hidden.Remove(other); + IReadOnlyList view = cached.GetTrialsPrefixView(studyId); + view.Should().BeOfType(); + view.Select(t => t.Number).Should().Equal(0, 1, 2, 3); + cached.GetAllTrials(studyId, deepcopy: false).Select(t => t.Number).Should().Equal(0, 1, 2, 3); + } + + /// + /// The genetic samplers read this view on every ask: a seeded NSGA-II study on the cached fake database is the + /// same study, trial by trial, as on an in-memory storage, and so is CMA-ES, which reads through the per-trial + /// cached list on this storage. + /// + /// nsga2 or cmaes. + [TestMethod] + [DataRow("nsga2")] + [DataRow("cmaes")] + public void SeededStudies_OnTheCachedStorage_MatchTheInMemoryRun(string samplerName) + { + string Run(BaseStorage storage) + { + BaseSampler sampler = samplerName == "nsga2" + ? new NSGAIISampler(populationSize: 10, seed: 7) + : new CmaEsSampler(seed: 7); + Study study = samplerName == "nsga2" + ? Optuna.CreateStudy(storage: storage, sampler: sampler, + directions: new[] { StudyDirection.Minimize, StudyDirection.Minimize }) + : Optuna.CreateStudy(storage: storage, sampler: sampler); + if (samplerName == "nsga2") + study.Optimize(t => new[] { t.SuggestFloat("x", -2, 2), 1 - t.SuggestFloat("y", -2, 2) }, nTrials: 120); + else + study.Optimize(t => Math.Pow(t.SuggestFloat("x", -2, 2), 2) + Math.Pow(t.SuggestFloat("y", -2, 2), 2), nTrials: 120); + return string.Join("\n", study.Trials.Select(Describe)); + } + + Run(new CachedStorage(new FakeDatabase())).Should().Be(Run(new InMemoryStorage())); + } + + /// A record's number, state, parameters (with value bits) and objective values, as one line. + /// The record. + /// The description. + private static string Describe(FrozenTrial trial) + => $"{trial.Number}|{trial.State}|" + + string.Join(",", trial.Params.Select(kv => $"{kv.Key}={(kv.Value is double d ? BitConverter.DoubleToInt64Bits(d) : kv.Value)}")) + + "|" + string.Join(",", (trial.Values ?? (IReadOnlyList)Array.Empty()).Select(BitConverter.DoubleToInt64Bits)); + + /// + /// A database stand-in for : an in-memory store whose reads for the cache return + /// deep copies, as rows read from a database are new objects, and which can hide trials from a sync. + /// + private sealed class FakeDatabase : InMemoryStorage, ICachedStorageBackend + { + /// Trial ids a synchronization does not return yet, as another worker's uncommitted trials. + internal HashSet Hidden { get; } = new(); + + /// + public FrozenTrial CreateNewTrialFull(int studyId, FrozenTrial? templateTrial = null) + => GetTrial(CreateNewTrial(studyId, templateTrial)).DeepCopy(); + + /// + public void SetTrialParamWithCachedDistribution( + int trialId, string paramName, double paramValueInternal, BaseDistribution distribution, + BaseDistribution? cachedDistribution) + => SetTrialParam(trialId, paramName, paramValueInternal, distribution); + + /// + public int GetBestTrialId(int studyId, StudyDirection direction) => GetBestTrial(studyId).TrialId; + + /// + public List GetTrialsPartial( + int studyId, IReadOnlyCollection? states, IReadOnlyCollection includedTrialIds, + int trialIdGreaterThan) + => GetAllTrials(studyId, deepcopy: true, states) + .Where(t => !Hidden.Contains(t.TrialId) + && (includedTrialIds.Contains(t.TrialId) || t.TrialId > trialIdGreaterThan)) + .ToList(); + + /// + public void RecordHeartbeat(int trialId) + { + } + + /// + public List GetStaleTrialIds(int studyId) => new(); + + /// + public int? GetHeartbeatInterval() => null; + + /// + public Action? GetHeartbeatStaleTrialCallback() => null; + } +} From ca9b9fed787ae97c2ecdf5e07eefc6815234e60c Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 01:11:36 +0200 Subject: [PATCH 013/145] docs(plan13): implementation record and before/after proof; Performance and Compatibility guide; README rerun Plan 13 (docs/plans/13-rustuna-derived-performance.md), phase 8: the documentation, and gate C's measurements. Before/after proof (gate C): one program (session 5417a82e, scratchpad/proof) built in Release against the de495f8 snapshot and against 656ab94; 40 float parameters, seed 0, sampler defaults; each workload before then after, back to back, on a machine other sessions loaded to 30-90 % CPU. Every pair hashed identically (every trial's parameters and values by bits): - CMA-ES 100,000 trials: 987.7 s -> 27.5 s (35.9x), allocations 519 GB -> 17.4 GB, peak working set 2.68 -> 1.88 GB; - CMA-ES 10,000: 25.4 -> 4.5 s (rep 0), 23.0 -> 2.6 s (rep 1); - NSGA-II 100,000: 73.2 -> 22.2 s, peak working set 1.53 -> 0.75 GB; NSGA-II 10,000: 3.8 -> 2.9 s / 2.7 -> 1.8 s; - TPE 1,000: 10.5 -> 5.0 s / 9.4 -> 4.1 s, gen-2 collections 254 -> 2; MOTPE 1,000: 47.4 -> 18.1 s; - QMC 10,000: 4.8 -> 2.5 s / 3.4 -> 1.05 s; Random 10,000: 5.8 -> 4.7 s / 4.9 -> 4.0 s. - The opt-in reuseToldOptimizer (D1): CMA-ES 10,000 2.0 s steady, 100,000 18.2 s; results differ, by design. docs/plans/13-rustuna-derived-performance.md: status BUILT except S8/S12; section 10 records each item, its commit, how it was gated and what was measured, the findings (the pickle's upper triangle behind D1, GetNTrials' hand-outs, dotnet-trace's suspensions showing up as lock and PollGC time), and the proof table. S9 is recorded as measured (attribute JSON 3-4 % of a cold load; the distribution parse cache built instead), S11 as measured and unchanged (the OpenBLAS thread count changes 100-D CMA-ES results, as it does NumPy's), S8 and S12 as left for the owner with their numbers, D2 as asked with both of its numbers (scoring is 41 % of TPE 1k and 49 % of MOTPE 1k after S6). docs/plans/README.md: plan 13 marked built; the workstream paragraph updated. docs/website-src/docs/performance.md (new, in the Guides TOC): what is fast and why results stay the same, the before/after table, the switches that change results (CmaEsSampler reuseToldOptimizer; TrialExecutorOptions.BlasThreads = 1 at nJobs 1) with whether seeded runs still match Python and whether Python workers can share the storage, and the thread-safety guarantee. docs/THREADING.md section 6.3: the plan 13 caches (CMA-ES restored optimizer and folded history, TPE split cache) are sampler state under the sampler monitor, one per PerWorker instance, keyed by what they were computed from. README.md, Performance: benchmark 14's C# probes rebuilt against 656ab94, three fresh processes each. Rows that moved beyond the busy machine's noise are updated (random over 1,000 parameters 13.5 -> 3.2 s, now faster than Python's 8.4 s, so the "one case where OptunaSharp is slower" sentence is gone; CMA-ES 200-D 2.8 -> 1.2 s; CMA-ES 6-D 1.75 -> 0.91 s; QMC 0.64 -> 0.39 s; random 6-D 1.34 -> 0.98 s; TPE 100-D 2.4 -> 2.1 s; the 50k blocks 0.6-0.8 s); the rest keep their numbers, and GP was checked interleaved against a de495f8 build (same within noise). A new "Large studies" table puts OptunaSharp next to Python Optuna 5.0 and Rustuna 0.2.1 on the announcement's workloads, with the measurement conditions stated (OptunaSharp's column ran on the busier machine). DocFX 2.78 builds the site with 0 errors; the new page adds no warning (the 1,160 warnings are the existing duplicate- notes and one old link in our-methods/nsga-iii-cpt.md). --- README.md | 42 ++++++++--- docs/THREADING.md | 6 ++ docs/plans/13-rustuna-derived-performance.md | 74 +++++++++++++++++++- docs/plans/README.md | 18 ++--- docs/website-src/docs/performance.md | 55 +++++++++++++++ docs/website-src/docs/toc.yml | 2 + 6 files changed, 178 insertions(+), 19 deletions(-) create mode 100644 docs/website-src/docs/performance.md diff --git a/README.md b/README.md index b5b4b43..8275a8e 100644 --- a/README.md +++ b/README.md @@ -257,17 +257,43 @@ sampler's own work dominates. OptunaSharp's times include the .NET JIT warming u |---|---|---| | TPE, 6 parameters, 500 trials | 1.12 s | 2.52 s | | TPE multivariate, 6 parameters, 500 trials | 1.14 s | 2.42 s | -| TPE, 100 parameters, 150 trials | 2.4 s | 19.1 s | +| TPE, 100 parameters, 150 trials | 2.1 s | 19.1 s | | GP, 6 parameters, 100 trials | 3.47 s | 17.55 s | -| CMA-ES, 6 parameters, 3,000 trials | 1.75 s | 5.21 s | -| CMA-ES, 200 parameters, 100 trials | 2.8 s | 3.3 s | +| CMA-ES, 6 parameters, 3,000 trials | 0.91 s | 5.21 s | +| CMA-ES, 200 parameters, 100 trials | 1.2 s | 3.3 s | | NSGA-II, 6 parameters, 3,000 trials | 0.60 s | 3.39 s | -| QMC, 6 parameters, 3,000 trials | 0.64 s | 3.49 s | -| Random, 6 parameters, 5,000 trials | 1.34 s | 1.36 s | -| Random, 1,000 parameters, 200 trials | 13.5 s | 8.4 s | -| Random, 4 parameters, 50,000 trials: each block of 10,000 | 0.7-0.9 s (the first 1.7 s) | 1.9-2.1 s | +| QMC, 6 parameters, 3,000 trials | 0.39 s | 3.49 s | +| Random, 6 parameters, 5,000 trials | 0.98 s | 1.36 s | +| Random, 1,000 parameters, 200 trials | 3.2 s | 8.4 s | +| Random, 4 parameters, 50,000 trials: each block of 10,000 | 0.6-0.8 s (the first 1.2 s) | 1.9-2.1 s | -Random sampling over 1,000 parameters is the one measured case where OptunaSharp is slower. +The rows for 100-parameter TPE, CMA-ES, QMC and Random were measured again on 2026-11-09 after the +[Rustuna-derived speedups](docs/plans/13-rustuna-derived-performance.md), which keep every seeded result; the +others moved less than the noise of a machine that was busy with other work at the time, so they keep their earlier +numbers. + +### Large studies + +40 float parameters in [-10, 10], seed 0, each sampler's defaults (TPE multivariate for one objective, independent +for two), the workloads of [Rustuna's announcement](https://medium.com/optuna/announcing-rustuna-cc82a6815bf7). +[Rustuna](https://github.com/optuna/rustuna) is the Optuna team's Rust port. OptunaSharp ran on 2026-11-09 on a +machine also busy with other work (30-90 % CPU load), the other two on 2026-11-08 on a quieter one, so OptunaSharp's +column errs on the slow side. OptunaSharp's times include the .NET JIT warming up. + +| Study | OptunaSharp | Python Optuna 5.0 | Rustuna 0.2.1 | +|---|---|---|---| +| TPE, 1,000 trials | 5.0 s | 35.2 s | 1.36 s | +| TPE, two objectives, 1,000 trials | 18.1 s | 163 s | 6.30 s | +| CMA-ES, 10,000 trials | 4.5 s | 56.9 s | 1.59 s | +| CMA-ES, 100,000 trials | 27.5 s | not run | 13.6 s | +| NSGA-II, 10,000 trials | 2.9 s | 44.2 s | 0.82 s | +| NSGA-II, 100,000 trials | 22.2 s | not run | 19.4 s | + +OptunaSharp's seeded trials are Optuna's; Rustuna's differ from the first trial (its own random generator). Where +OptunaSharp is still behind Rustuna, part of the gap is the protocol it keeps so Python workers can share a study: +CMA-ES pickles its state into the storage every generation, and TPE scores candidates in NumPy's order. The +[Performance and Compatibility](docs/website-src/docs/performance.md) guide lists the opt-in switches that trade +that compatibility for speed. ### Executors diff --git a/docs/THREADING.md b/docs/THREADING.md index 856ef53..8cf3549 100644 --- a/docs/THREADING.md +++ b/docs/THREADING.md @@ -608,6 +608,12 @@ and ends, around one native call to OpenBLAS). - Per-study state lives in `PerStudyState`, keyed by the storage object (held weakly) and the study id, because study ids repeat across storages. `TrialAnchor` detects a deleted study whose id was reused. The class is not synchronized itself: the sampler monitor covers it. +- The performance caches of plan 13 are sampler state of the same kind, unsynchronized and covered by the + monitor: `CmaEsSampler`'s restored optimizer and its folded history (`CmaTrialHistory`, which starts + over when it is handed another storage, study or history), and `TPESampler`'s last trial split + (`TpeSplitCache`). A `SamplerConcurrency.PerWorker` worker instance gets its own of each, so workers + never share one. Each is keyed by what it was computed from (the holder trial's id, the very records + split), so a cache that outlives its study only misses. - Random numbers come from `LazyRandomState`, MT19937, never `System.Random`. An unseeded sampler seeds itself from OS entropy (`RandomNumberGenerator`) rather than the clock. Samplers created at the same moment therefore draw different streams: in one process, in worker processes, and in parallel diff --git a/docs/plans/13-rustuna-derived-performance.md b/docs/plans/13-rustuna-derived-performance.md index 9b46b42..18686b7 100644 --- a/docs/plans/13-rustuna-derived-performance.md +++ b/docs/plans/13-rustuna-derived-performance.md @@ -1,8 +1,10 @@ # Plan 13 — Rustuna-derived performance: compatible speedups first, opt-in paths where results would change -> **Status: DESIGN (2026-11-08).** Nothing here is built yet. A scratch prototype of the CMA-ES part exists (session -> `5417a82e`, a patched copy of `d0558f5`) and is measured below; Phase 1 ports it. Background, benchmarks and the -> full Rustuna read-through: catalog note `optunasharp/notes/rustuna-comparison-2026-11-08.md`. +> **Status: BUILT (2026-11-09), except S8 and S12 (§10).** Phases 1-4 are on `main`: `8f8ce3e` (S1-S3, D1), +> `605806c` (S4, S7), `dce5885` (S3, S5, S6), `656ab94` (S10, S9 as measured). S11 was measured and changes nothing; +> D2 awaits the owner with both of its numbers. §10 records what was built, how it was gated, and the before/after +> measurements. Background, benchmarks and the full Rustuna read-through: catalog note +> `optunasharp/notes/rustuna-comparison-2026-11-08.md`. --- @@ -346,3 +348,69 @@ protocol is what lets Python workers share OptunaSharp's studies. - D3 (trial queue) and D4 (discarding) after S12's memory numbers. - D2's default once S6 and its fast variant are measured. - §5.5's switch shape for future switches; D1 uses a constructor parameter. + +## 10. Implementation record (2026-11-08/09) + +Every item below keeps OptunaSharp's results to the bit unless it says otherwise. The gates, each run per phase: +the unit suite's CI filter on net8.0 and net10.0; the live-Python interop suite on net10.0 with the engine required; +the documentation gate; a before/after hash of seeded studies (every trial's parameters and values by bits) built from +the `de495f8` snapshot and the working tree; and, for the concurrency-sensitive items, `ConcurrencyGun` cases. + +| Item | Commit | What was built | Measured | +|---|---|---|---| +| S1, S2 | `8f8ce3e` | `CmaTrialHistory` folds finished trials once; the optimizer restored from a state is cached by its holder trial and dropped after each tell | CMA-ES 40-D 100k trials: see the proof table | +| S3 | `8f8ce3e`, `dce5885` | `Study.GetTrialsForSampler` (zero-copy view in memory, Optuna's read elsewhere) for the intersection space and CMA-ES; TPE's constant-liar read skips the current trial without reading it; QMC stops at the first finished trial instead of sorting; the group-decomposed space filters while scanning | QMC 3,000 trials 2.23 → 1.23 s | +| D1 | `8f8ce3e` | `CmaEsSampler(..., reuseToldOptimizer: false)`, opt-in | ~15 % on 40-D; results differ from generation 3 (§2.4) | +| S4 | `605806c` | `InMemoryStorage` writes a running trial in place while no reader holds it (`FrozenTrial.StorageExclusive`, `HandOut`, `WritableTrialUnlocked`); the view reads unfinished records under the lock | 40,000 MOTPE suggestions: 71.5 ms of `SetTrialParam` in total | +| S7 | `605806c` | Bitwise-identical float/int distributions and parameter names shared per study | CMA-ES 100k peak working set ~2.5 GB → 1.44 GB (with S4) | +| S5 | `dce5885` | `TpeSplitCache`: the last split reused for the very same records | MOTPE 1,000: one split per trial instead of 40 | +| S6 | `dce5885` | Fused continuous log-pdf (per-pair precomputation, `Vector` lanes per kernel, scalar order kept), pooled grids and matrix, span `LogGaussMass`/`Ndtr`/`Erf.ComputeArray`, `StableArgsort`, presized `GetInternalRepr` | TPE 1,000: 10.08 → 3.93 s steady; MOTPE 1,000: 41.07 → 15.77 s | +| S10 | `656ab94` | `CachedStorage` keeps its cache in number order (`ByNumber`, placeholders for unsynced numbers); `GetAllTrials` stops sorting and `GetTrialsPrefixView` wraps the cache | O(n log n) → O(n) per read, O(1) view | +| S9 | `656ab94` | Measured first: attribute JSON is 3-4 % of a 10k-trial SQLite cold load, so lazy decoding was not built. Parsing each `distribution_json` row was ~40 %: both RDB storages now parse each distinct text once per load (`JournalDistributionCache`) | Cold load 2.1 → 1.4-1.5 s, 1.22 GB → 479 MB | +| S11 | none | Measured: at 10-D and 40-D CMA-ES and 6-D GP the OpenBLAS thread count changes no result, but at 100-D every count gives a different (deterministic) result, as NumPy's does. Pinning one thread at `nJobs: 1` would break parity with Python's default-thread NumPy, so the automatic policy stays (pin only when `nJobs > 1`). One thread was 12 % faster at 40-D and 24-35 % at 100-D: available through `TrialExecutorOptions.BlasThreads = 1`, at the cost of parity | — | +| S8 | not built | The bulk attribute API adds public surface Optuna lacks; after S4 an in-memory attribute write is already in place (O(1)), and the measured database cost is dominated by per-suggestion parameter writes, which no attribute API can batch. Left for the owner | — | +| S12 | not built | Measured instead: S4 and S7 took the 100k-trial CMA-ES study from ~2.5 GB to 1.44 GB peak working set. The rest is per-trial dictionaries and boxed values plus the generation pickles (UTF-16 strings); compacting them means materializing `FrozenTrial` contents on read, a larger change with its own risks. Left for the owner with these numbers (and D4 with it) | — | +| D2 | pending | Scoring still takes 41 % of a 1,000-trial TPE run and 49 % of a 1,000-trial MOTPE run after S6 (profiles, 2026-11-09). A fast kernel could at best roughly halve that share; §2.4's noise harness bounds the decision risk. Asked of the owner with these numbers | — | + +**Findings along the way.** +- The CMA-ES pickle stores only C's upper triangle; keeping the told optimizer instead of a restore is therefore + observable (§2.4), which is why S1 drops its cache after every tell and D1 is opt-in. +- `GetNTrials` in Optuna's base storage counts through `GetAllTrials(deepcopy: false)`, so a count hands out every + running record; kept as Optuna has it (nothing in `src/` calls it). +- Profiles taken with `dotnet-trace`'s sampled thread time put whole seconds into `Monitor.Enter`'s slow path and + `PollGC` in single-threaded runs: the sampler's thread suspensions surface there. A stopwatch around + `SetTrialParam` measured 1.8 µs per call where the profile implied 66 µs. +- `OptunaHubBridgeParityTests.BenchmarkBridge_ToyProblem_ExposesSpaceAndOptimizes` asserted a bound on an unseeded TPE + study and failed in a few percent of runs; it is seeded now (`dce5885`). + +### 10.1 Before/after proof + +Gate C. `scratchpad/proof/Program.cs` (session `5417a82e`) built twice in Release: against the `de495f8` snapshot +(before) and against `656ab94` (after). 40 float parameters in [-10, 10], seed 0, sampler defaults; objective +`sum((x - 1)^2)`, or `sum(x^2)` and `sum((x - 2)^2)` for two objectives. i9-13900K, Windows 11, .NET 10, power throttling +opted out, OpenBLAS at its default thread count. Each workload ran before then after, back to back, 2026-11-09 +00:39-01:03, while other sessions loaded the machine to 30-90 % CPU, so both columns are slower than §2.1's quieter +run (CMA-ES 100k before: 988 s here, 812 s there). Rep 0 includes the JIT; rep 1 is a second study in the same +process. The hash covers every trial's parameters and values by their bits. + +| Workload | Before rep 0 / rep 1 | After rep 0 / rep 1 | Speed-up | Allocated before → after | Peak working set | Hash | +|---|---|---|---|---|---|---| +| Random, 10,000 | 5.79 / 4.92 s | 4.72 / 3.96 s | 1.2× | 3.6 → 2.3 GB | 216 → 256 MB | same | +| QMC, 10,000 | 4.76 / 3.43 s | 2.47 / 1.05 s | 1.9× / 3.3× | 3.7 → 1.6 GB | 285 → 258 MB | same | +| NSGA-II, 10,000 | 3.82 / 2.75 s | 2.88 / 1.77 s | 1.3× / 1.6× | 2.9 → 1.2 GB | 230 → 205 MB | same | +| TPE, 1,000 | 10.53 / 9.37 s | 5.01 / 4.11 s | 2.1× / 2.3× | 4.4 → 1.6 GB (gen-2 GCs 254 → 2) | 98 → 107 MB | same | +| MOTPE, 1,000 | 47.42 s | 18.07 s | 2.6× | 28.7 → 3.4 GB (gen-2 594 → 2) | 82 → 79 MB | same | +| CMA-ES, 10,000 | 25.35 / 23.05 s | 4.49 / 2.64 s | 5.6× / 8.7× | 10.9 → 1.8 GB | 340 → 317 MB | same | +| NSGA-II, 100,000 | 73.17 s | 22.16 s | 3.3× | 63.4 → 12.4 GB | 1,527 → 750 MB | same | +| CMA-ES, 100,000 | 987.71 s | 27.49 s | 35.9× | 519 → 17.4 GB | 2,680 → 1,876 MB | same | +| CMA-ES, 10,000, `reuseToldOptimizer: true` (D1) | — | 3.43 / 2.04 s | 1.3× over after | 1.5 GB | 304 MB | differs, by design | +| CMA-ES, 100,000, `reuseToldOptimizer: true` (D1) | — | 18.18 s | 1.5× over after | 15.2 GB | 1,758 MB | differs, by design | + +Against Rustuna 0.2.1 (§2.1, quieter machine): TPE 1k 1.36 s, MOTPE 1k 6.30 s, CMA-ES 10k 1.59 s and 100k 13.6 s, +NSGA-II 10k 0.82 s and 100k 19.4 s. The §7 targets were "about half" for NSGA-II and MOTPE and 2.6 / 26 s for CMA-ES: +met or beaten except CMA-ES 10k's first rep, which carries the JIT (2.64 s steady). + +The README's sampler table (benchmark 14's C# probes, rebuilt against `656ab94`, three fresh processes each) moved +where the change is beyond the busy machine's noise: random over 1,000 parameters 13.5 → 3.2 s (now faster than +Python's 8.4 s), CMA-ES 200-D 2.8 → 1.2 s, CMA-ES 6-D 1.75 → 0.91 s, QMC 0.64 → 0.39 s, random 6-D 1.34 → 0.98 s, +TPE 100-D 2.4 → 2.1 s. GP was rerun interleaved against a `de495f8` build: the same within noise. diff --git a/docs/plans/README.md b/docs/plans/README.md index dcf7be2..26a7903 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -46,7 +46,7 @@ numeric prefix is the recommended execution order. | [10](10-threading.md) | Threading: a configurable `nJobs` executor, then seven follow-ups | the reused worker threads of `9cfcb12` (done) | N | ★★ / medium | | [11](11-threading-round-2.md) | Threading, round 2: sampler per worker, OpenBLAS threads, locked log handlers, the async question | plan 10 (done) | N | ★★ / medium | | [12](12-parallel-read-duplicates.md) | Parallel reads never hand one point to two trials: per-sampler fixes and tests, the bridges pinned | plan 11 (done) | N | ★★ / medium | -| [13](13-rustuna-derived-performance.md) | Rustuna-derived performance: compatible speedups as the only path, opt-in switches where results would change | plans 10-12 (done) | N | ★★★ / high | +| [13](13-rustuna-derived-performance.md) | **Built (2026-11-09)** — Rustuna-derived performance: compatible speedups as the only path, opt-in switches where results would change | plans 10-12 (done) | N | ★★★ / high | **Threading workstream (10, 11, 12).** Unrelated to JANUS. Plan 12 (done 2026-11-08) makes every Core sampler give trials that ask at once different points, without changing sequential runs, and pins what the four Python bridges @@ -58,13 +58,15 @@ through an options class at four levels. Phase 2, done only after Phase 1, close [`docs/THREADING.md`](../THREADING.md) §12: OS seed width, the terminator lock, Redis round trips and `DedicatedThreads`, cancellation, `PymooSampler` on any thread, and ML.NET search under `nJobs`. -**Performance workstream (13).** Designed with the owner on 2026-11-08, not started. Rustuna, the Optuna team's -Rust port, runs its announcement's workloads 3-60x faster than OptunaSharp. Plan 13 adopts each of its techniques that -keeps every result identical as the only implementation; a scratch prototype took CMA-ES at 100,000 trials from 812 s -to 26 s that way, byte-identical. The techniques that would change results are measured, and the owner decides each -one: CMA-ES reusing its told optimizer is opt-in with Optuna's path as the default; TPE fast numerics waits for the exact -fused kernel; the trial queue and trial discarding are deferred, with compact storage of finished trials proposed -instead of discarding. +**Performance workstream (13).** Designed with the owner on 2026-11-08; built 2026-11-09 (`8f8ce3e`, `605806c`, +`dce5885`, `656ab94`) except two items left for the owner. Rustuna, the Optuna team's Rust port, ran its announcement's +workloads 3-60x faster than OptunaSharp. Plan 13 adopts each of its techniques that keeps every result identical as +the only implementation: seeded studies hash identically before and after, and CMA-ES at 100,000 trials went from +988 s to 27.5 s, TPE at 1,000 trials from 10.5 s to 5.0 s, NSGA-II at 100,000 from 73 s to 22 s (§10 of the plan). +CMA-ES reusing its told optimizer is opt-in with Optuna's path as the default; the OpenBLAS thread count was measured +and left as it is (it changes results from about 100 dimensions); lazy attribute decoding was measured and replaced by +parsing each distribution text once. Left for the owner with numbers: TPE fast numerics, the bulk attribute API, and +compact storage of finished trials; the trial queue and trial discarding stay deferred. **Tier-X pymoo-interop workstream (08–09).** Separate from the Tier-N native ports above: unlocked by the 2026-09-10 pymoo-on-numpy-2.4.2 migration (pymoo now runs in the interop interpreter, commit `9c7aec2`), diff --git a/docs/website-src/docs/performance.md b/docs/website-src/docs/performance.md new file mode 100644 index 0000000..c1a56a8 --- /dev/null +++ b/docs/website-src/docs/performance.md @@ -0,0 +1,55 @@ +# Performance and Compatibility + +OptunaSharp reproduces Optuna's seeded results: the same seed, sampler and objective give the same trials, value for +value, as Python Optuna 5.0. Speed work keeps that promise. A technique that leaves every result unchanged is simply +built in; one that would change results is a switch on the component it changes, off by default, and this page lists +each one. + +## What is fast, and why results stay the same + +The optimizations below came from comparing OptunaSharp with [Rustuna](https://github.com/optuna/rustuna), the Optuna +team's Rust port (plan 13 in the repository, `docs/plans/13-rustuna-derived-performance.md`). None of them is +observable: seeded studies hash identically, every trial's parameters and values by their bits, before and after. + +| Area | What changed | Effect | +|---|---|---| +| `CmaEsSampler` | The optimizer restored from the stored state is kept for the rest of its generation, and finished trials are scanned once instead of on every ask | 40 parameters, 100,000 trials: about 35 times faster | +| `InMemoryStorage` | A running trial nobody has read is written in place; a record any read returned is copied before its next write, so what a caller holds never changes | Suggestions no longer copy the trial's dictionaries | +| `InMemoryStorage` | Identical distributions and parameter names are stored once per study | Less memory for long studies | +| `TPESampler` | The scoring kernel computes per (kernel, dimension) what it used to compute per candidate, on vector lanes that keep the scalar summation order, with pooled buffers | With the next row: 40 parameters, 1,000 trials about twice as fast, and 2.6 times with two objectives | +| `TPESampler` | Independent sampling (the default for several objectives) reuses the trial's below/above split for every parameter | One split per trial instead of one per parameter | +| `QMCSampler`, genetic samplers, search spaces | History read through a zero-copy view instead of a copied list | Per-ask cost no longer grows with the study | +| `CachedStorage` (around every RDB storage) | Cached trials kept in number order instead of sorted on every read | | +| SQLite and EF Core storages | Each distinct distribution text parsed once per load | SQLite, 10,000-trial cold load: 2.1 s to 1.5 s, 60 % less memory | + +Measured on an Intel Core i9-13900K under Windows 11 with .NET 10 Release builds, 40 float parameters, seed 0, the +same program built against the code before and after; each pair ran back to back on a machine that was also running +other work, so compare within a row. The first number of a pair includes the .NET JIT warming up. + +| Study | Before | After | Same results | +|---|---|---|---| +| TPE, 1,000 trials | 10.5 s | 5.0 s | yes | +| TPE (two objectives), 1,000 trials | 47.4 s | 18.1 s | yes | +| CMA-ES, 10,000 trials | 25.4 s | 4.5 s | yes | +| CMA-ES, 100,000 trials | 988 s | 27.5 s | yes | +| NSGA-II, 10,000 trials | 3.8 s | 2.9 s | yes | +| NSGA-II, 100,000 trials | 73.2 s | 22.2 s | yes | +| QMC, 10,000 trials | 4.8 s | 2.5 s | yes | +| Random, 10,000 trials | 5.8 s | 4.7 s | yes | + +## Switches that change results + +| Switch | What it does | Seeded runs still match Python? | Can Python workers share the storage? | +|---|---|---|---| +| `new CmaEsSampler(..., reuseToldOptimizer: true)` | Keeps the optimizer that just told a generation instead of restoring the state it stored; about 20-25 % faster on 40 parameters | No: from about the third generation (Optuna's pickle stores only the covariance's upper triangle, so a restored optimizer differs in the last bits) | Yes: the state is still stored every generation exactly as Optuna stores it | +| `TrialExecutorOptions.BlasThreads = 1` with `nJobs: 1` | Runs OpenBLAS on one thread; CMA-ES was 12 % faster at 40 parameters and 24-35 % faster at 100 | Up to about 40 parameters, yes; at 100 parameters no, because OpenBLAS's threaded kernels sum in another order (NumPy's results depend on the thread count the same way) | Yes | + +With `nJobs` above 1 OptunaSharp already runs OpenBLAS on one thread automatically: parallel runs reseed every trial, +so they are not reproducible anyway, and several workers in linear algebra at once would otherwise oversubscribe the +cores (see [Easy Parallelization](key-features/parallelization.md)). + +## Thread safety + +None of the caches changes what concurrent callers see. A record read from an in-memory storage, through any read, is +never changed afterwards; the samplers' caches are covered by the sampler's lock like the rest of a sampler's state, +and each `SamplerConcurrency.PerWorker` worker has its own. The repository's `docs/THREADING.md` has the details. diff --git a/docs/website-src/docs/toc.yml b/docs/website-src/docs/toc.yml index 8430859..615cf00 100644 --- a/docs/website-src/docs/toc.yml +++ b/docs/website-src/docs/toc.yml @@ -46,6 +46,8 @@ href: logging.md - name: Exceptions href: exceptions.md + - name: Performance and Compatibility + href: performance.md - name: Recipes items: - name: Overview From 05edfdf40fe7d5061215cbccefcf85f99cd819fd Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 07:45:56 +0200 Subject: [PATCH 014/145] docs(tpe): name the overload in three crefs the span overloads of dce5885 made ambiguous dce5885 (plan 13, item S6: the fused TPE kernel) added span overloads TruncNorm.LogGaussMass(ReadOnlySpan, ReadOnlySpan, Span) and Erf.ComputeArray(ReadOnlySpan, Span) next to the array ones. Three doc comments still named the methods without a signature: - src/OptunaSharp.Core/Internal/TruncNorm.cs:21 (class remarks) -> Erf.ComputeArray(double[]), the array overload the sentence is about (Optuna's >= 2000-element vectorized erf); - src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs:491-492 (the grid precompute's remarks) -> the span overloads, which is what that code calls. A plain build does not show this: the projects keep NoWarn for the doc warnings, and only the documentation workflow (.github/workflows/documentation.yml) rebuilds with -p:NoWarn= and CS0419 among WarningsAsErrors. Found by running that gate with --no-incremental; an incremental rebuild skips CoreCompile and reports 0 errors, so the gate has to run on a fresh compile. No code change. --- src/OptunaSharp.Core/Internal/TruncNorm.cs | 2 +- src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/OptunaSharp.Core/Internal/TruncNorm.cs b/src/OptunaSharp.Core/Internal/TruncNorm.cs index f7e1f3a..60cd6ce 100644 --- a/src/OptunaSharp.Core/Internal/TruncNorm.cs +++ b/src/OptunaSharp.Core/Internal/TruncNorm.cs @@ -18,7 +18,7 @@ namespace OptunaSharp.Internal; /// the plan already excuses for LAPACK-backed linear algebra; the categorical path, the trial /// selection, and everything not routed through erf stay bit-exact. Optuna's own ≥2000- /// element vectorized erf approximation IS reproduced bit-for-bit by -/// . +/// . /// internal static class TruncNorm { diff --git a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs index e4ccc06..a6510e4 100644 --- a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs +++ b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs @@ -488,8 +488,8 @@ internal double[] LogPdf(double[,] x) /// /// /// The grids are laid out by dimension ([c * n_kernels + k]) so the kernel loop is contiguous. - /// is elementwise, and the vectorized-erf switch depends only - /// on how many elements it gets (see ), so evaluating the same elements in + /// is elementwise, and the vectorized-erf switch depends only + /// on how many elements it gets (see ), so evaluating the same elements in /// another order changes no element. /// /// From 53a7a0f97dd395dddb0b2f577aabe3112674c609 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 07:45:57 +0200 Subject: [PATCH 015/145] feat(storage): write several attributes in one step (plan 13, item S8), with Optuna-named overloads Rustuna's storages take a batch of attributes (set_trial_user_attrs and friends) and write it under one lock / in one transaction. Optuna has no such call: a caller loops over set_trial_user_attr, and every call takes the storage lock, checks the trial, and (in a database) commits on its own. The owner chose to build it (2026-11-09: "Build it", "Also have optuna's api overload"), so the batch exists under Rustuna's plural names and under Optuna's singular names as overloads. The result never differs from the loop: every override leaves exactly what calling the single-key method once per entry, in the dictionary's enumeration order, would leave. What changes is the cost, and that a rejected batch writes nothing. API (all documented with params, exceptions and remarks): - BaseStorage: virtual SetStudyUserAttrs / SetStudySystemAttrs / SetTrialUserAttrs / SetTrialSystemAttrs(id, IReadOnlyDictionary). The default body is the loop, so every third-party storage supports the call unchanged (atomic only as far as each single call). Non-virtual Optuna-name overloads SetStudyUserAttr(id, dict), SetStudySystemAttr(id, dict), SetTrialUserAttr(id, dict), SetTrialSystemAttr(id, dict) forward to them. - BaseTrial.SetUserAttrs(dict) (virtual, default loop) + SetUserAttr(dict) overload; Trial.SetUserAttrs overrides it: one storage call, then the trial's own frozen view and the thread-cache mirror take every entry, exactly as SetUserAttr(key, value) updates them per key. - Study.SetUserAttrs(dict) + SetUserAttr(dict) overload -> storage.SetStudyUserAttrs. - Empty dictionary: writes nothing and checks nothing (no KeyError for an unknown id), in every override; null throws ArgumentNullException. Per storage: - InMemoryStorage: one lock, one existence/updatable check, and at most one record copy for the whole batch (WritableTrialUnlocked from S4: in place while no reader holds the record). Study attributes: one ShallowClone of the dictionary, every entry written into it, published once. - JournalStorage: the batch is N ordinary SINGLE-KEY records appended with one AppendLogs. Not one record with N keys: Python's replay asserts `len(log["user_attr"]) == 1` (optuna/storages/journal/_storage.py, _apply_set_trial_user_attr and siblings), so a multi-key record would make the file unreadable to Optuna. Under the thread lock: sync, check the trial (new JournalStorageReplayResult.ThrowIfTrialNotUpdatable: KeyError NotFoundMsg / UpdateFinishedTrialError UnupdatableMsgFormat, the replay's own messages) or the study (ThrowIfStudyMissing), stamp the worker id, append, then SyncAfterBatch. SyncAfterBatch exists because the replay throws for a rejected record OF THIS WORKER after advancing past it, leaving the rest of the batch unread: it syncs up to batch-size times, keeps the first exception and rethrows it after the batch is drained, so one rejected record (another process finished the trial between our check and our append) does not poison the next call. Test hooks: AppendBatchAndSyncForTests, WorkerIdForTests (internal). - CachedStorage: forwards each batch whole to the backend (its own cache is read-through for attributes, as for the single calls). - SQLite RDBStorage: one transaction per batch: the study or trial (FindTrialOrRaise + CheckTrialIsUpdatableOrRaise) is checked once, every entry upserted with the same SQL as the single call, one commit. A failure rolls back the whole batch. - EF Core RDBStorage: the same through Mutate(ctx => transaction; check; add/update each entry; SaveChanges; Commit). Docs: three crefs that named BaseTrial.SetUserAttr / Study.SetUserAttr without a signature became ambiguous with the new overload (ComponentLosses.cs, FrozenStudy.cs, StudySummary.cs, and one in JanusMooCmixSamplerTests.cs); they now name (string, object?). The documentation gate (check_documentation.cs over src: 0 issues; Core rebuilt --no-incremental with the workflow's WarningsAsErrors: 0 errors) passes. Tests: - test/OptunaSharp.Tests/Storages/BulkAttributeWritesTests.cs (new, 14 cases): batch == loop for in-memory, journal and cached storage (trial user/system, study user/system, overwrite order); the Optuna-name overloads; empty and null dictionaries; finished/unknown trial and unknown study write nothing and throw the single call's error; Trial.SetUserAttrs updates the trial's own view and a later frozen copy; a storage that does not override gets the loop (DelegatingStorage test double); the journal writes N single-key records with one worker id; and the drain test (a rejected own record followed by valid ones: the call throws the first error and the next call works). The drain test was checked against a mutation (SyncAfterBatch reduced to one sync): it fails there. - test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs (+3): batch leaves what single writes leave (read back through a fresh storage), all-or-nothing on a finished trial, and the batch through CachedStorage reaches the database. - test/OptunaSharp.Tests.Interop/JournalInteropTests.cs (+1, live Python): CSharp_WritesAttributeBatches_Python_ReadsEveryOne: C# writes a trial user batch (int, float, string, bool, None, list), a trial system batch, a study user batch and a study system batch to a journal file; live Optuna 5.0 loads the study, reads every attribute with its value (json.dumps compared byte-for-byte), continues the study with two trials, and C# reloads all three. Runs: full CI unit filter (TestCategory!=OpenBugs&!=HighMemory&!=RequiresPython&!=Slow) on both TFMs: 2070 passed, 1 skipped, 0 failed (net8.0 and net10.0). Storage suite cases and the interop case pass. --- src/OptunaSharp.Core/BaseTrial.cs | 28 ++ src/OptunaSharp.Core/ComponentLosses.cs | 2 +- src/OptunaSharp.Core/FrozenStudy.cs | 2 +- src/OptunaSharp.Core/Storages/BaseStorage.cs | 121 +++++ .../Storages/CachedStorage.cs | 31 ++ .../Storages/InMemoryStorage.cs | 85 ++++ .../Storages/Journal/JournalStorage.cs | 174 +++++++ src/OptunaSharp.Core/Study.cs | 17 + src/OptunaSharp.Core/StudySummary.cs | 2 +- src/OptunaSharp.Core/Trial.cs | 34 ++ .../Storages/Rdb/RDBStorage.cs | 112 +++++ .../Storages/Rdb/RDBStorage.cs | 84 ++++ .../JournalInteropTests.cs | 56 +++ .../SqliteRdbStorageTests.cs | 102 ++++ .../Samplers/JanusMooCmixSamplerTests.cs | 2 +- .../Storages/BulkAttributeWritesTests.cs | 464 ++++++++++++++++++ 16 files changed, 1312 insertions(+), 4 deletions(-) create mode 100644 test/OptunaSharp.Tests/Storages/BulkAttributeWritesTests.cs diff --git a/src/OptunaSharp.Core/BaseTrial.cs b/src/OptunaSharp.Core/BaseTrial.cs index 905df50..5c7002c 100644 --- a/src/OptunaSharp.Core/BaseTrial.cs +++ b/src/OptunaSharp.Core/BaseTrial.cs @@ -231,6 +231,34 @@ public TEnum SuggestCategorical(string name, IEnumerable choices) /// Stores a user attribute by name. public abstract void SetUserAttr(string key, object? value); + /// + /// Stores several user attributes at once (Rustuna's set_user_attrs; Rustuna plan 13, item S8). Optuna has + /// no such call: the result is what once per entry, in the dictionary's + /// order, leaves. + /// + /// The attributes, set in enumeration order; persistent backends require JSON-serializable values. + /// is null. + /// Whatever throws for an entry. + /// + /// This default calls per entry. overrides it + /// to write the whole batch through one storage call (one lock, journal append or transaction), so an unknown or + /// finished trial gets nothing written. + /// + public virtual void SetUserAttrs(IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + foreach (var (key, value) in attrs) + SetUserAttr(key, value); + } + + /// + /// Optuna's set_user_attr name for : several user attributes in one call. + /// + /// The attributes, set in enumeration order. + /// is null. + /// Whatever throws. + public void SetUserAttr(IReadOnlyDictionary attrs) => SetUserAttrs(attrs); + /// /// Expose this trial's observed-components vector — the per-trial residual terms /// r = (r_1, …, r_m) of an objective that decomposes as f(x) = mean_i r_i(x) — on diff --git a/src/OptunaSharp.Core/ComponentLosses.cs b/src/OptunaSharp.Core/ComponentLosses.cs index 3d996c7..fc73287 100644 --- a/src/OptunaSharp.Core/ComponentLosses.cs +++ b/src/OptunaSharp.Core/ComponentLosses.cs @@ -91,7 +91,7 @@ public static bool TryRead(IReadOnlyDictionary userAttrs, out d /// /// Validate a to-be-written component vector against the channel contract and return an owned - /// copy suitable for . Rejects an empty vector and any + /// copy suitable for . Rejects an empty vector and any /// non-finite entry (so a stored vector always reads back — a non-finite one never silently /// becomes "absent"). Negative entries are accepted (read at their zero clamp downstream). /// diff --git a/src/OptunaSharp.Core/FrozenStudy.cs b/src/OptunaSharp.Core/FrozenStudy.cs index c65acc1..f7e02f0 100644 --- a/src/OptunaSharp.Core/FrozenStudy.cs +++ b/src/OptunaSharp.Core/FrozenStudy.cs @@ -51,7 +51,7 @@ public FrozenStudy( /// /// Dictionary that contains the attributes of the set with - /// . + /// . /// public OrderedDict UserAttrs { get; internal set; } diff --git a/src/OptunaSharp.Core/Storages/BaseStorage.cs b/src/OptunaSharp.Core/Storages/BaseStorage.cs index 70046b7..3bb9b52 100644 --- a/src/OptunaSharp.Core/Storages/BaseStorage.cs +++ b/src/OptunaSharp.Core/Storages/BaseStorage.cs @@ -126,6 +126,64 @@ internal T TellLocked(int trialId, Func body) /// Attribute value; persistent backends require JSON-serializable data. public abstract void SetStudySystemAttr(int studyId, string key, object? value); + /// + /// Register several user-defined attributes to a study at once, overwriting existing values (Rustuna's + /// set_study_user_attrs; Rustuna plan 13, item S8). Optuna has no such call: the result is what calling + /// once per entry, in the dictionary's order, leaves. + /// + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order; persistent backends require JSON-serializable values. + /// is null. + /// If no study with the matching ID exists. + /// + /// This default writes one entry per call, exactly as a caller looping over + /// would, so any storage supports it; it is atomic only as far as each single call is. The built-in storages + /// override it to write in one step: one lock in memory, one appended batch in a journal, one transaction in a + /// database. An empty dictionary writes nothing and checks nothing. + /// + public virtual void SetStudyUserAttrs(int studyId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + foreach (var (key, value) in attrs) + SetStudyUserAttr(studyId, key, value); + } + + /// + /// Register several optuna-internal attributes to a study at once (Rustuna's set_study_system_attrs; + /// plan 13, item S8): the result of once per entry, in order. + /// + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order; persistent backends require JSON-serializable values. + /// is null. + /// If no study with the matching ID exists. + /// The default writes one entry per call; see for what the overrides do. + public virtual void SetStudySystemAttrs(int studyId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + foreach (var (key, value) in attrs) + SetStudySystemAttr(studyId, key, value); + } + + /// + /// Optuna's set_study_user_attr name for : several user-defined attributes + /// in one call. + /// + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// is null. + /// If no study with the matching ID exists. + public void SetStudyUserAttr(int studyId, IReadOnlyDictionary attrs) => SetStudyUserAttrs(studyId, attrs); + + /// + /// Optuna's set_study_system_attr name for : several optuna-internal + /// attributes in one call. + /// + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// is null. + /// If no study with the matching ID exists. + public void SetStudySystemAttr(int studyId, IReadOnlyDictionary attrs) => SetStudySystemAttrs(studyId, attrs); + // ------------------------------------------------------------------ study access /// Read the ID of a study. @@ -265,6 +323,69 @@ public virtual double GetTrialParam(int trialId, string paramName) /// Attribute value; persistent backends require JSON-serializable data. public abstract void SetTrialSystemAttr(int trialId, string key, object? value); + /// + /// Set several user-defined attributes of a trial at once, overwriting existing values (Rustuna's + /// set_trial_user_attrs; Rustuna plan 13, item S8). Optuna has no such call: the result is what calling + /// once per entry, in the dictionary's order, leaves. + /// + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order; persistent backends require JSON-serializable values. + /// is null. + /// If no trial with the matching ID exists. + /// If the trial is already finished. + /// + /// This default writes one entry per call, as a caller's own loop would, so any storage supports it; a failure + /// part-way leaves the entries before it written. The built-in storages override it to write in one step and + /// check the trial once, so a finished or unknown trial gets nothing written: one lock in memory, one appended + /// batch of ordinary single-attribute records in a journal (what Python's journal reader expects), one + /// transaction in a database. An empty dictionary writes nothing and checks nothing. + /// + public virtual void SetTrialUserAttrs(int trialId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + foreach (var (key, value) in attrs) + SetTrialUserAttr(trialId, key, value); + } + + /// + /// Set several optuna-internal attributes of a trial at once (Rustuna's set_trial_system_attrs; plan 13, + /// item S8): the result of once per entry, in order. + /// + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order; persistent backends require JSON-serializable values. + /// is null. + /// If no trial with the matching ID exists. + /// If the trial is already finished. + /// The default writes one entry per call; see for what the overrides do. + public virtual void SetTrialSystemAttrs(int trialId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + foreach (var (key, value) in attrs) + SetTrialSystemAttr(trialId, key, value); + } + + /// + /// Optuna's set_trial_user_attr name for : several user-defined attributes + /// in one call. + /// + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// is null. + /// If no trial with the matching ID exists. + /// If the trial is already finished. + public void SetTrialUserAttr(int trialId, IReadOnlyDictionary attrs) => SetTrialUserAttrs(trialId, attrs); + + /// + /// Optuna's set_trial_system_attr name for : several optuna-internal + /// attributes in one call. + /// + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// is null. + /// If no trial with the matching ID exists. + /// If the trial is already finished. + public void SetTrialSystemAttr(int trialId, IReadOnlyDictionary attrs) => SetTrialSystemAttrs(trialId, attrs); + // ------------------------------------------------------------------ trial access /// Read a trial. diff --git a/src/OptunaSharp.Core/Storages/CachedStorage.cs b/src/OptunaSharp.Core/Storages/CachedStorage.cs index 0b728b7..215009c 100644 --- a/src/OptunaSharp.Core/Storages/CachedStorage.cs +++ b/src/OptunaSharp.Core/Storages/CachedStorage.cs @@ -192,6 +192,20 @@ public override void SetStudyUserAttr(int studyId, string key, object? value) public override void SetStudySystemAttr(int studyId, string key, object? value) => _backendStorage.SetStudySystemAttr(studyId, key, value); + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// + /// Forwarded whole, so the backend writes the batch in one step (plan 13, item S8). + public override void SetStudyUserAttrs(int studyId, IReadOnlyDictionary attrs) + => _backendStorage.SetStudyUserAttrs(studyId, attrs); + + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// + /// Forwarded whole, so the backend writes the batch in one step (plan 13, item S8). + public override void SetStudySystemAttrs(int studyId, IReadOnlyDictionary attrs) + => _backendStorage.SetStudySystemAttrs(studyId, attrs); + /// Name identifying the study in the selected storage. /// public override int GetStudyIdFromName(string studyName) @@ -340,6 +354,23 @@ public override void SetTrialUserAttr(int trialId, string key, object? value) public override void SetTrialSystemAttr(int trialId, string key, object? value) => _backendStorage.SetTrialSystemAttr(trialId, key: key, value: value); + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// + /// + /// Forwarded whole, so the backend writes the batch in one step (plan 13, item S8); like the single-attribute + /// write, it leaves the cache alone (unfinished trials are re-read from the backend on the next sync). + /// + public override void SetTrialUserAttrs(int trialId, IReadOnlyDictionary attrs) + => _backendStorage.SetTrialUserAttrs(trialId, attrs); + + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// + /// Forwarded whole, as . + public override void SetTrialSystemAttrs(int trialId, IReadOnlyDictionary attrs) + => _backendStorage.SetTrialSystemAttrs(trialId, attrs); + /// Storage identifier of the trial; distinct from its per-study trial number. /// Looks up a finished trial without consulting the backend. /// A cached finished trial, or null if the record is missing or unfinished. diff --git a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs index 992f7ac..cb73637 100644 --- a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs +++ b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs @@ -116,6 +116,47 @@ public override void SetStudySystemAttr(int studyId, string key, object? value) } } + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// + /// + /// One lock and one new dictionary for the whole batch (plan 13, item S8), published only after every entry is + /// in it, so a reader sees all of them or none, and a reader holding the previous dictionary sees no change. + /// + public override void SetStudyUserAttrs(int studyId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + lock (_lock) + { + CheckStudyId(studyId); + var userAttrs = _studies[studyId].UserAttrs.ShallowClone(); + foreach (var (key, value) in attrs) + userAttrs[key] = value; + _studies[studyId].UserAttrs = userAttrs; + } + } + + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// + /// One lock and one new dictionary for the whole batch, as . + public override void SetStudySystemAttrs(int studyId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + lock (_lock) + { + CheckStudyId(studyId); + var systemAttrs = _studies[studyId].SystemAttrs.ShallowClone(); + foreach (var (key, value) in attrs) + systemAttrs[key] = value; + _studies[studyId].SystemAttrs = systemAttrs; + } + } + /// Name identifying the study in the selected storage. /// public override int GetStudyIdFromName(string studyName) @@ -539,6 +580,50 @@ public override void SetTrialSystemAttr(int trialId, string key, object? value) } } + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// + /// + /// One lock, one updatable check and at most one copy of the record for the whole batch (plan 13, item S8): an + /// unknown or finished trial gets nothing written. + /// + public override void SetTrialUserAttrs(int trialId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + lock (_lock) + { + var trial = GetTrialUnlocked(trialId); + CheckTrialIsUpdatable(trialId, trial.State); + + // One writable record for every entry: in place unless a reader may hold it (WritableTrialUnlocked). + var userAttrs = WritableTrialUnlocked(trialId).UserAttrsDict; + foreach (var (key, value) in attrs) + userAttrs[key] = value; + } + } + + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// + /// One lock, one check and at most one copy, as . + public override void SetTrialSystemAttrs(int trialId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + lock (_lock) + { + var trial = GetTrialUnlocked(trialId); + CheckTrialIsUpdatable(trialId, trial.State); + + var systemAttrs = WritableTrialUnlocked(trialId).SystemAttrsDict; + foreach (var (key, value) in attrs) + systemAttrs[key] = value; + } + } + /// Storage identifier of the trial; distinct from its per-study trial number. /// public override FrozenTrial GetTrial(int trialId) diff --git a/src/OptunaSharp.Core/Storages/Journal/JournalStorage.cs b/src/OptunaSharp.Core/Storages/Journal/JournalStorage.cs index e7761e0..2c01f22 100644 --- a/src/OptunaSharp.Core/Storages/Journal/JournalStorage.cs +++ b/src/OptunaSharp.Core/Storages/Journal/JournalStorage.cs @@ -607,6 +607,152 @@ public override void SetTrialSystemAttr(int trialId, string key, object? value) } } + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// + /// See : one checked, appended batch of ordinary records. + public override void SetStudyUserAttrs(int studyId, IReadOnlyDictionary attrs) + => WriteAttributeBatch(JournalOperation.SetStudyUserAttr, "study_id", studyId, "user_attr", attrs, isTrial: false); + + /// Storage identifier of the study; distinct from its name. + /// The attributes, written in enumeration order. + /// + /// See : one checked, appended batch of ordinary records. + public override void SetStudySystemAttrs(int studyId, IReadOnlyDictionary attrs) + => WriteAttributeBatch(JournalOperation.SetStudySystemAttr, "study_id", studyId, "system_attr", attrs, isTrial: false); + + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// + /// See : one checked, appended batch of ordinary records. + public override void SetTrialUserAttrs(int trialId, IReadOnlyDictionary attrs) + => WriteAttributeBatch(JournalOperation.SetTrialUserAttr, "trial_id", trialId, "user_attr", attrs, isTrial: true); + + /// Storage identifier of the trial; distinct from its per-study trial number. + /// The attributes, written in enumeration order. + /// + /// See : one checked, appended batch of ordinary records. + public override void SetTrialSystemAttrs(int trialId, IReadOnlyDictionary attrs) + => WriteAttributeBatch(JournalOperation.SetTrialSystemAttr, "trial_id", trialId, "system_attr", attrs, isTrial: true); + + /// + /// The bulk attribute writes (plan 13, item S8): one record per entry, exactly the record the single-attribute + /// write appends, all appended in one backend call. + /// + /// The single-attribute operation each record carries. + /// The record's id field: study_id or trial_id. + /// The study or trial id. + /// The record's attribute field: user_attr or system_attr. + /// The attributes, one record each, in enumeration order. + /// Whether is a trial (checked as updatable) or a study (checked as existing). + /// is null. + /// The study or trial does not exist. + /// The trial has finished. + /// A value cannot be serialized, or the backend cannot append; nothing is appended then. + /// + /// + /// One record per entry, on purpose. Python's journal replay asserts that an attribute record carries + /// exactly one key (assert len(log["user_attr"]) == 1), so a journal Python workers share can only hold + /// single-key records. The batch is those records, appended together: the file backend writes them under one + /// lock with one flush, where single writes pay one each. + /// + /// + /// Checked first. The replay raises for this worker's own record on an unknown or finished trial, one + /// record per synchronization. Appending a whole batch the replay then rejects would leave records that make + /// later, unrelated calls of this storage raise, so the batch is checked against the replayed state first and + /// nothing is written when the check fails. If another process finishes the trial between the check and the + /// append, consumes the batch's rejected records before the error surfaces. + /// + /// + private void WriteAttributeBatch( + JournalOperation opCode, string idField, int id, string attrField, + IReadOnlyDictionary attrs, bool isTrial) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + + var logs = new List>(attrs.Count); + foreach (var (key, value) in attrs) + { + logs.Add(new OrderedDict + { + ["op_code"] = (int)opCode, + ["worker_id"] = null, // filled under the lock: the worker id belongs to the replay state + [idField] = id, + [attrField] = new OrderedDict { [key] = value }, + }); + } + + lock (_threadLock) + { + SyncWithBackend(); + if (isTrial) + _replayResult.ThrowIfTrialNotUpdatable(id); + else + _replayResult.ThrowIfStudyMissing(id); + + string workerId = _replayResult.WorkerId; + foreach (var log in logs) + log["worker_id"] = workerId; + _backend.AppendLogs(logs); + SyncAfterBatch(logs.Count); + } + } + + /// + /// Synchronizes after appending a batch of this worker's records, consuming every record of the batch the replay + /// rejects before raising the first rejection. + /// + /// How many records the batch appended; each rejection consumes at least one of them. + /// The first exception the synchronization raised, rethrown with its original stack. + /// + /// A rejection here means another process finished or deleted the target between the check and the append. The + /// replay raises once per rejected record, and only for the worker that wrote it, so without draining, each of + /// the batch's remaining records would raise in a later call of this storage that has nothing to do with it. + /// Each failed synchronization advances past at least the record that failed, so at most one attempt per record + /// is needed. + /// + private void SyncAfterBatch(int batchSize) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo? first = null; + for (int attempt = 0; attempt < batchSize; attempt++) + { + try + { + SyncWithBackend(); + break; + } + catch (Exception exception) + { + first ??= System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(exception); + } + } + first?.Throw(); + } + + /// Test hook for : appends records as one batch, then drains as it does. + /// Complete records, worker id included. + /// The first rejection of the batch, after the batch is consumed. + internal void AppendBatchAndSyncForTests(IReadOnlyList> logs) + { + lock (_threadLock) + { + _backend.AppendLogs(logs); + SyncAfterBatch(logs.Count); + } + } + + /// The worker id this storage writes into records, for tests that build records of their own. + internal string WorkerIdForTests + { + get + { + lock (_threadLock) + return _replayResult.WorkerId; + } + } + /// Storage identifier of the trial; distinct from its per-study trial number. /// public override FrozenTrial GetTrial(int trialId) @@ -1155,6 +1301,34 @@ private void ApplySetTrialSystemAttr(OrderedDict log) } } + /// + /// Raises what replaying this worker's own attribute record for would raise, before + /// any record is written: the bulk attribute writes (plan 13, item S8) check once, then append their batch. + /// + /// Storage identifier of the trial about to be written. + /// The trial does not exist (). + /// The trial has finished; the message is the replay's. + internal void ThrowIfTrialNotUpdatable(int trialId) + { + if (!_trials.TryGetValue(trialId, out FrozenTrial? trial)) + throw new KeyError(JournalStorage.NotFoundMsg); + if (trial.State.IsFinished()) + throw new UpdateFinishedTrialError( + string.Format(CultureInfo.InvariantCulture, JournalStorage.UnupdatableMsgFormat, trial.Number)); + } + + /// + /// Raises what replaying this worker's own study-attribute record would raise for a missing study, before any + /// record is written (plan 13, item S8). + /// + /// Storage identifier of the study about to be written. + /// The study does not exist (). + internal void ThrowIfStudyMissing(int studyId) + { + if (!_studies.ContainsKey(studyId)) + throw new KeyError(JournalStorage.NotFoundMsg); + } + /// Storage identifier of the trial; distinct from its per-study trial number. /// Decoded journal operation, including its payload and originating worker identifier. /// Checks trial existence and completion state before applying a replay mutation. diff --git a/src/OptunaSharp.Core/Study.cs b/src/OptunaSharp.Core/Study.cs index 53a7d4f..185c15f 100644 --- a/src/OptunaSharp.Core/Study.cs +++ b/src/OptunaSharp.Core/Study.cs @@ -692,6 +692,23 @@ public TrialsTable TrialsDataframe(string[]? attrs = null, bool multiIndex = fal /// A value of the attribute. The value should be JSON serializable. public void SetUserAttr(string key, object? value) => Storage.SetStudyUserAttr(StudyId, key, value); + /// + /// Set several user attributes to the study at once (Rustuna's set_user_attrs; plan 13, item S8), with one + /// storage call: the stored result is what once per entry leaves. + /// + /// The attributes, set in enumeration order. The values should be JSON serializable. + /// is null. + /// The study no longer exists in storage; nothing is set. + /// A configured storage fails; its exception propagates. + public void SetUserAttrs(IReadOnlyDictionary attrs) => Storage.SetStudyUserAttrs(StudyId, attrs); + + /// Optuna's set_user_attr name for : several user attributes in one call. + /// The attributes, set in enumeration order. The values should be JSON serializable. + /// is null. + /// The study no longer exists in storage; nothing is set. + /// A configured storage fails; its exception propagates. + public void SetUserAttr(IReadOnlyDictionary attrs) => SetUserAttrs(attrs); + // Study.set_system_attr (deprecated 3.1.0) was removed in Optuna 5.0.0 (#6834). Optuna-internal // writers go through Storage.SetStudySystemAttr(StudyId, key, value); users keep SetUserAttr. diff --git a/src/OptunaSharp.Core/StudySummary.cs b/src/OptunaSharp.Core/StudySummary.cs index e7ac268..d000e6c 100644 --- a/src/OptunaSharp.Core/StudySummary.cs +++ b/src/OptunaSharp.Core/StudySummary.cs @@ -59,7 +59,7 @@ public StudySummary( /// with best objective value in the . public FrozenTrial? BestTrial { get; } - /// Dictionary that contains the attributes of the study set with . + /// Dictionary that contains the attributes of the study set with . public OrderedDict UserAttrs { get; } /// The number of trials ran in the . diff --git a/src/OptunaSharp.Core/Trial.cs b/src/OptunaSharp.Core/Trial.cs index ecdd8af..8d09e75 100644 --- a/src/OptunaSharp.Core/Trial.cs +++ b/src/OptunaSharp.Core/Trial.cs @@ -361,6 +361,40 @@ public override void SetUserAttr(string key, object? value) } } + /// + /// Set several user attributes to the trial at once (Rustuna's set_user_attrs; plan 13, item S8), with one + /// storage call: the stored and cached result is what once per entry + /// leaves. + /// + /// The attributes, set in enumeration order. The values should be JSON serializable. + /// is null. + /// The storage record is already finished; nothing is set. + /// The trial was removed from storage; nothing is set. + /// A configured storage fails; its exception propagates. + /// + /// The storage writes the batch in one step (), and the trial's + /// own copy is updated only after the storage accepted it, so a refused batch leaves both unchanged. Under the + /// trial's monitor, like , so a concurrent read of this trial's + /// attributes sees the batch whole or not at all. + /// + public override void SetUserAttrs(IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + using (MonitorGate.Scope(_syncRoot)) + { + Storage.SetTrialUserAttrs(TrialId, attrs); + foreach (var (key, value) in attrs) + _cachedFrozenTrial.UserAttrsDict[key] = value; + MirrorIntoThreadCache(t => + { + foreach (var (key, value) in attrs) + t.UserAttrsDict[key] = value; + }); + } + } + // Trial.set_system_attr (deprecated 3.1.0) was removed in Optuna 5.0.0 (#6834). Optuna-internal // writers call Storage.SetTrialSystemAttr(TrialId, key, value) directly; users keep SetUserAttr. diff --git a/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs b/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs index 8f192a7..6a04432 100644 --- a/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs +++ b/src/OptunaSharp.Storage.EFCore/Storages/Rdb/RDBStorage.cs @@ -610,6 +610,61 @@ public override void SetStudyUserAttr(int studyId, string key, object? value) public override void SetStudySystemAttr(int studyId, string key, object? value) => SetStudyAttr(isUser: false, studyId, key, value); + /// + /// One transaction and one save for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetStudyUserAttrs(int studyId, IReadOnlyDictionary attrs) + => SetStudyAttrs(isUser: true, studyId, attrs); + + /// + /// One transaction and one save for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetStudySystemAttrs(int studyId, IReadOnlyDictionary attrs) + => SetStudyAttrs(isUser: false, studyId, attrs); + + /// + /// Upserts several JSON-serializable study attributes in one transaction after verifying once that the study + /// exists: the rows writes per entry, saved together. + /// + /// Whether to use user-attribute records instead of system-attribute records. + /// The storage-wide study identifier, not its display name. + /// The attributes, upserted in enumeration order. + /// is null. + /// The study does not exist; nothing is written. + /// A value is not JSON-serializable or the database refuses the batch; nothing is written. + private void SetStudyAttrs(bool isUser, int studyId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + Mutate(ctx => + { + // Disposed without Commit on any exception, which rolls the whole batch back. + using var tx = ctx.Database.BeginTransaction(); + EnsureStudyExists(ctx, studyId); + foreach (var (key, value) in attrs) + { + string json = PyJson.Dumps(value); + if (isUser) + { + var row = ctx.StudyUserAttributes.FirstOrDefault(a => a.StudyId == studyId && a.Key == key); + if (row is null) + ctx.StudyUserAttributes.Add(new StudyUserAttributeEntity { StudyId = studyId, Key = key, ValueJson = json }); + else + row.ValueJson = json; + } + else + { + var row = ctx.StudySystemAttributes.FirstOrDefault(a => a.StudyId == studyId && a.Key == key); + if (row is null) + ctx.StudySystemAttributes.Add(new StudySystemAttributeEntity { StudyId = studyId, Key = key, ValueJson = json }); + else + row.ValueJson = json; + } + } + ctx.SaveChanges(); + tx.Commit(); + }); + } + /// Upserts a JSON-serializable attribute after verifying that its study exists. /// Whether to use user-attribute records instead of system-attribute records. /// The storage-wide study identifier, not its display name. @@ -1104,6 +1159,63 @@ public override void SetTrialUserAttr(int trialId, string key, object? value) public override void SetTrialSystemAttr(int trialId, string key, object? value) => SetTrialAttr(isUser: false, trialId, key, value); + /// + /// One transaction and one save for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetTrialUserAttrs(int trialId, IReadOnlyDictionary attrs) + => SetTrialAttrs(isUser: true, trialId, attrs); + + /// + /// One transaction and one save for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetTrialSystemAttrs(int trialId, IReadOnlyDictionary attrs) + => SetTrialAttrs(isUser: false, trialId, attrs); + + /// + /// Upserts several JSON-serializable trial attributes in one transaction after checking once that the trial is + /// still mutable: the rows writes per entry, saved together. + /// + /// Whether to use user-attribute records instead of system-attribute records. + /// The storage-wide trial identifier, distinct from its study-local number. + /// The attributes, upserted in enumeration order. + /// is null. + /// The trial does not exist; nothing is written. + /// The trial has finished; nothing is written. + /// A value is not JSON-serializable or the database refuses the batch; nothing is written. + private void SetTrialAttrs(bool isUser, int trialId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + Mutate(ctx => + { + // Disposed without Commit on any exception, which rolls the whole batch back. + using var tx = ctx.Database.BeginTransaction(); + var trial = FindTrialForUpdate(ctx, trialId); + CheckTrialIsUpdatableOrRaise(trial); + foreach (var (key, value) in attrs) + { + string json = PyJson.Dumps(value); + if (isUser) + { + var row = ctx.TrialUserAttributes.FirstOrDefault(a => a.TrialId == trialId && a.Key == key); + if (row is null) + ctx.TrialUserAttributes.Add(new TrialUserAttributeEntity { TrialId = trialId, Key = key, ValueJson = json }); + else + row.ValueJson = json; + } + else + { + var row = ctx.TrialSystemAttributes.FirstOrDefault(a => a.TrialId == trialId && a.Key == key); + if (row is null) + ctx.TrialSystemAttributes.Add(new TrialSystemAttributeEntity { TrialId = trialId, Key = key, ValueJson = json }); + else + row.ValueJson = json; + } + } + ctx.SaveChanges(); + tx.Commit(); + }); + } + /// Upserts a JSON-serializable attribute after verifying that its trial is still mutable. /// Whether to use user-attribute records instead of system-attribute records. /// The storage-wide trial identifier, distinct from its study-local number. diff --git a/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs b/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs index 09d6c97..5ae09ed 100644 --- a/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs +++ b/src/OptunaSharp.Storage.Sqlite/Storages/Rdb/RDBStorage.cs @@ -569,6 +569,47 @@ private void SetStudyAttr(string table, int studyId, string key, object? value) } } + /// + /// One transaction for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetStudyUserAttrs(int studyId, IReadOnlyDictionary attrs) + => SetStudyAttrs("study_user_attributes", studyId, attrs); + + /// + /// One transaction for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetStudySystemAttrs(int studyId, IReadOnlyDictionary attrs) + => SetStudyAttrs("study_system_attributes", studyId, attrs); + + /// + /// Upserts several JSON-serializable attributes in one transaction after verifying once that the study exists: + /// the same rows writes per entry. + /// + /// The trusted schema table name selected by the storage implementation. + /// The storage-wide study identifier, not its display name. + /// The attributes, upserted in enumeration order. + /// is null. + /// The study does not exist; nothing is written. + /// A value is not JSON-serializable; the transaction rolls back and nothing is written. + private void SetStudyAttrs(string table, int studyId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + lock (_lock) + { + // Disposed without Commit on any exception, which rolls every upsert of the batch back. + using var tx = _conn.BeginTransaction(); + EnsureStudyExists(studyId, tx); + foreach (var (key, value) in attrs) + { + Exec( + $"INSERT INTO {table} (study_id, \"key\", value_json) VALUES ($p0, $p1, $p2) " + + "ON CONFLICT(study_id, \"key\") DO UPDATE SET value_json = excluded.value_json", + tx, studyId, key, PyJson.Dumps(value)); + } + tx.Commit(); + } + } + /// /// No study has the requested name. public override int GetStudyIdFromName(string studyName) @@ -1003,6 +1044,49 @@ private void SetTrialAttr(string table, int trialId, string key, object? value) } } + /// + /// One transaction for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetTrialUserAttrs(int trialId, IReadOnlyDictionary attrs) + => SetTrialAttrs("trial_user_attributes", trialId, attrs); + + /// + /// One transaction for the whole batch (plan 13, item S8): all entries are stored or none are. + public override void SetTrialSystemAttrs(int trialId, IReadOnlyDictionary attrs) + => SetTrialAttrs("trial_system_attributes", trialId, attrs); + + /// + /// Upserts several JSON-serializable trial attributes in one transaction after checking once that the trial is + /// still mutable: the same rows writes per entry. + /// + /// The trusted schema table name selected by the storage implementation. + /// The storage-wide trial identifier, distinct from its study-local number. + /// The attributes, upserted in enumeration order. + /// is null. + /// The trial does not exist; nothing is written. + /// The trial has finished; nothing is written. + /// A value is not JSON-serializable; the transaction rolls back and nothing is written. + private void SetTrialAttrs(string table, int trialId, IReadOnlyDictionary attrs) + { + ArgumentNullException.ThrowIfNull(attrs); + if (attrs.Count == 0) + return; + lock (_lock) + { + // Disposed without Commit on any exception, which rolls every upsert of the batch back. + using var tx = _conn.BeginTransaction(); + var trial = FindTrialOrRaise(trialId, tx); + CheckTrialIsUpdatableOrRaise(trial); + foreach (var (key, value) in attrs) + { + Exec( + $"INSERT INTO {table} (trial_id, \"key\", value_json) VALUES ($p0, $p1, $p2) " + + "ON CONFLICT(trial_id, \"key\") DO UPDATE SET value_json = excluded.value_json", + tx, trialId, key, PyJson.Dumps(value)); + } + tx.Commit(); + } + } + /// Upserts a trial attribute inside the caller-owned transaction without committing it. /// The current SQLite transaction, or null to execute outside an explicit transaction. /// The trusted schema table name selected by the storage implementation. diff --git a/test/OptunaSharp.Tests.Interop/JournalInteropTests.cs b/test/OptunaSharp.Tests.Interop/JournalInteropTests.cs index 914ea61..f76ffa1 100644 --- a/test/OptunaSharp.Tests.Interop/JournalInteropTests.cs +++ b/test/OptunaSharp.Tests.Interop/JournalInteropTests.cs @@ -179,4 +179,60 @@ public void CSharp_Writes_First_Python_Continues() Optuna.LoadStudy("cs-first", NewCSharpStorage(), sampler: new RandomSampler(4)) .Trials.Should().HaveCount(5); } + + /// + /// Plan 13, item S8: attributes C# writes in batches (a trial's user and system attributes, a study's user and + /// system attributes) are ordinary single-key journal records, so live Python Optuna replays the file without + /// tripping its assert len(log["user_attr"]) == 1, sees every attribute with its value, and continues the + /// study in the same file. + /// + [TestMethod] + [TestCategory("RequiresPython")] + public void CSharp_WritesAttributeBatches_Python_ReadsEveryOne() + { + PythonSession.EnsureOrInconclusive(); + + var storage = NewCSharpStorage(); + var study = Optuna.CreateStudy(storage: storage, studyName: "batches", sampler: new RandomSampler(seed: 5)); + var batch = new OptunaSharp.Collections.OrderedDict + { + ["seed"] = 7L, ["ratio"] = 0.25, ["name"] = "run", ["flag"] = true, ["none"] = null, + ["list"] = new System.Collections.Generic.List { 1L, "two" }, + }; + Trial trial = study.Ask(); + trial.SuggestFloat("x", 0, 1); + trial.SetUserAttrs(batch); + storage.SetTrialSystemAttrs(trial.TrialId, new OptunaSharp.Collections.OrderedDict { ["sys"] = "s" }); + study.SetUserAttrs(batch); + storage.SetStudySystemAttrs(study.StudyId, new OptunaSharp.Collections.OrderedDict { ["ssys"] = 1L }); + study.Tell(trial, 0.5); + + using (Py.GIL()) + { + using PyModule scope = Py.CreateScope(); + scope.Exec( + "import json, optuna\n" + + "optuna.logging.set_verbosity(optuna.logging.WARNING)\n" + + $"path = r'{_path}'\n" + + "storage = optuna.storages.JournalStorage(optuna.storages.journal.JournalFileBackend(" + + "path, lock_obj=optuna.storages.journal.JournalFileOpenLock(path)))\n" + + "study = optuna.load_study(study_name='batches', storage=storage)\n" + + "t0 = study.trials[0]\n" + + "trial_user = json.dumps(t0.user_attrs, sort_keys=True)\n" + + "trial_sys = json.dumps(storage.get_trial_system_attrs(t0._trial_id), sort_keys=True)\n" + + "study_user = json.dumps(study.user_attrs, sort_keys=True)\n" + + "study_sys = json.dumps(storage.get_study_system_attrs(study._study_id), sort_keys=True)\n" + + "study.optimize(lambda t: t.suggest_float('x', 0, 1), n_trials=2)\n" + + "n_trials = len(study.trials)\n"); + + string expected = "{\"flag\": true, \"list\": [1, \"two\"], \"name\": \"run\", \"none\": null, \"ratio\": 0.25, \"seed\": 7}"; + scope.Get("trial_user").As().Should().Be(expected); + scope.Get("study_user").As().Should().Be(expected); + scope.Get("trial_sys").As().Should().Contain("\"sys\": \"s\""); + scope.Get("study_sys").As().Should().Contain("\"ssys\": 1"); + scope.Get("n_trials").As().Should().Be(3); + } + + Optuna.LoadStudy("batches", NewCSharpStorage()).Trials.Should().HaveCount(3); + } } diff --git a/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs b/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs index 90e3997..5b10ad6 100644 --- a/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs +++ b/test/OptunaSharp.Tests.Storage/SqliteRdbStorageTests.cs @@ -56,6 +56,108 @@ private string TempDbPath() private static OrderedDict QuadDist() => new() { ["x"] = new FloatDistribution(-10, 10) }; + // ------------------------------------------------------------------ plan 13: bulk attribute writes (S8) + + /// + /// Plan 13, item S8: a batch of trial and study attributes on SQLite leaves exactly what one single write per + /// entry leaves (keys, values and their JSON), read back through a fresh storage on the same file. + /// + [TestMethod] + public void BulkAttributes_LeaveWhatSingleWritesLeave() + { + string path = TempDbPath(); + string url = $"sqlite:///{path.Replace('\\', '/')}"; + var attrs = new OrderedDict + { + ["seed"] = 7L, ["ratio"] = 0.25, ["name"] = "run", ["flag"] = true, ["none"] = null, + ["list"] = new List { 1L, "two" }, + }; + int bulkStudy, singleStudy, bulkTrial, singleTrial; + using (var storage = new RDBStorage(url)) + { + bulkStudy = storage.CreateNewStudy(new[] { StudyDirection.Minimize }, "bulk"); + singleStudy = storage.CreateNewStudy(new[] { StudyDirection.Minimize }, "single"); + bulkTrial = storage.CreateNewTrial(bulkStudy); + singleTrial = storage.CreateNewTrial(singleStudy); + storage.SetTrialUserAttr(bulkTrial, "name", "earlier"); + storage.SetTrialUserAttr(singleTrial, "name", "earlier"); + + storage.SetTrialUserAttrs(bulkTrial, attrs); + storage.SetTrialSystemAttrs(bulkTrial, attrs); + storage.SetStudyUserAttrs(bulkStudy, attrs); + storage.SetStudySystemAttrs(bulkStudy, attrs); + foreach (var (key, value) in attrs) + { + storage.SetTrialUserAttr(singleTrial, key, value); + storage.SetTrialSystemAttr(singleTrial, key, value); + storage.SetStudyUserAttr(singleStudy, key, value); + storage.SetStudySystemAttr(singleStudy, key, value); + } + } + + using var reopened = new RDBStorage(url); + static string Describe(IReadOnlyDictionary a) + => string.Join("; ", a.OrderBy(kv => kv.Key, StringComparer.Ordinal).Select(kv => $"{kv.Key}={PyJson.Dumps(kv.Value)}")); + Describe(reopened.GetTrial(bulkTrial).UserAttrs).Should().Be(Describe(reopened.GetTrial(singleTrial).UserAttrs)); + Describe(reopened.GetTrial(bulkTrial).SystemAttrs).Should().Be(Describe(reopened.GetTrial(singleTrial).SystemAttrs)); + Describe(reopened.GetStudyUserAttrs(bulkStudy)).Should().Be(Describe(reopened.GetStudyUserAttrs(singleStudy))); + Describe(reopened.GetStudySystemAttrs(bulkStudy)).Should().Be(Describe(reopened.GetStudySystemAttrs(singleStudy))); + reopened.GetTrial(bulkTrial).UserAttrs["name"].Should().Be("run", "the batch overwrites an earlier value"); + } + + /// + /// A batch is one transaction: a value that cannot be serialized, in the middle of the batch, rolls back the + /// entries before it; a finished trial and an unknown study refuse the batch with nothing written. + /// + [TestMethod] + public void BulkAttributes_AreAllOrNothing() + { + using RDBStorage storage = NewMemoryStorage(); + int studyId = storage.CreateNewStudy(new[] { StudyDirection.Minimize }, "atomic"); + int trialId = storage.CreateNewTrial(studyId); + var poisoned = new OrderedDict { ["ok"] = 1L, ["bad"] = new object(), ["after"] = 2L }; + + Action trialBatch = () => storage.SetTrialUserAttrs(trialId, poisoned); + Action studyBatch = () => storage.SetStudyUserAttrs(studyId, poisoned); + trialBatch.Should().Throw(); + studyBatch.Should().Throw(); + storage.GetTrial(trialId).UserAttrs.Should().BeEmpty("the transaction rolled the first entry back"); + storage.GetStudyUserAttrs(studyId).Should().BeEmpty(); + + storage.SetTrialStateValues(trialId, TrialState.Complete, new[] { 1.0 }); + Action finished = () => storage.SetTrialSystemAttrs(trialId, new OrderedDict { ["x"] = 1L }); + Action unknownStudy = () => storage.SetStudySystemAttrs(424242, new OrderedDict { ["x"] = 1L }); + finished.Should().Throw(); + unknownStudy.Should().Throw(); + storage.GetTrial(trialId).SystemAttrs.Should().NotContainKey("x"); + } + + /// + /// A study on SQLite runs through , which forwards a trial's and a study's batch to the + /// backend whole; the trial and the study see their attributes, and so does a fresh storage. + /// + [TestMethod] + public void BulkAttributes_ThroughCachedStorage_ReachTheDatabase() + { + string path = TempDbPath(); + string url = $"sqlite:///{path.Replace('\\', '/')}"; + var rdb = new RDBStorage(url); + Study study = Optuna.CreateStudy(storage: rdb, studyName: "cached"); + study.Storage.Should().BeOfType(); + Trial trial = study.Ask(); + trial.SetUserAttrs(new OrderedDict { ["a"] = 1L, ["b"] = "two" }); + study.SetUserAttr(new OrderedDict { ["c"] = 3.5 }); + study.Tell(trial, 1.0); + + trial.UserAttrs.Keys.Should().Contain(new[] { "a", "b" }); + study.UserAttrs["c"].Should().Be(3.5); + rdb.Dispose(); + using var reopened = new RDBStorage(url); + int studyId = reopened.GetStudyIdFromName("cached"); + reopened.GetAllTrials(studyId)[0].UserAttrs.Should().Contain(new KeyValuePair("b", "two")); + reopened.GetStudyUserAttrs(studyId)["c"].Should().Be(3.5); + } + // ------------------------------------------------------------------ plan 13: one parse per distribution text /// diff --git a/test/OptunaSharp.Tests/Samplers/JanusMooCmixSamplerTests.cs b/test/OptunaSharp.Tests/Samplers/JanusMooCmixSamplerTests.cs index 72678ac..2836b6c 100644 --- a/test/OptunaSharp.Tests/Samplers/JanusMooCmixSamplerTests.cs +++ b/test/OptunaSharp.Tests/Samplers/JanusMooCmixSamplerTests.cs @@ -172,7 +172,7 @@ double Run() /// /// ZDT1 over four parameters that stores its residual vector [f1, f2] under /// . The canonical key goes through the typed writer users are told to - /// call, any other key through , and null stores nothing. + /// call, any other key through , and null stores nothing. /// /// The trial being evaluated. /// User-attribute key for the residual vector, or null to store none. diff --git a/test/OptunaSharp.Tests/Storages/BulkAttributeWritesTests.cs b/test/OptunaSharp.Tests/Storages/BulkAttributeWritesTests.cs new file mode 100644 index 0000000..ab88d0b --- /dev/null +++ b/test/OptunaSharp.Tests/Storages/BulkAttributeWritesTests.cs @@ -0,0 +1,464 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using AwesomeAssertions; +using NumSharp; +using OptunaSharp.Collections; +using OptunaSharp.Distributions; +using OptunaSharp.Internal; +using OptunaSharp.Storages; +using OptunaSharp.Storages.Journal; + +namespace OptunaSharp.Tests.Storages; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S8, the owner's choice of 2026-11-09: bulk attribute +/// writes under Rustuna's names (SetTrialUserAttrs, SetStudyUserAttrs, ...) and Optuna's +/// (SetUserAttr(IReadOnlyDictionary) overloads). A batch leaves exactly what one single write per entry leaves; +/// the built-in storages write it in one step and check first, so an unknown or finished target gets nothing; a +/// journal batch is ordinary single-key records, which Python's journal reader requires. +/// +[TestClass] +public class BulkAttributeWritesTests +{ + private static readonly StudyDirection[] Minimize = { StudyDirection.Minimize }; + + private readonly List _paths = new(); + + /// Deletes the journal files the tests made; a file still held open is left for the temp cleanup. + [TestCleanup] + public void TearDown() + { + foreach (string path in _paths) + { + foreach (string file in Directory.GetFiles(Path.GetDirectoryName(path)!, Path.GetFileName(path) + "*")) + { + try + { + File.Delete(file); + } + catch (IOException) + { + } + } + } + } + + /// Attribute values of every JSON kind, in an order the batch must keep. + /// The attributes. + private static OrderedDict SampleAttrs() => new() + { + ["seed"] = 7L, + ["ratio"] = 0.25, + ["name"] = "run-a", + ["flag"] = true, + ["missing"] = null, + ["list"] = new List { 1L, "two", 3.5 }, + ["nested"] = new OrderedDict { ["k"] = new List { false } }, + }; + + /// A fresh storage of the named kind: memory, journal (a file), or default (the base loop). + /// The storage kind. + /// The storage. + /// is not a known kind. + private BaseStorage Open(string kind) + { + switch (kind) + { + case "memory": + return new InMemoryStorage(); + case "journal": + { + string path = Path.Combine(Path.GetTempPath(), $"optunasharp-bulk-{Guid.NewGuid():N}.log"); + _paths.Add(path); + return new JournalStorage(new JournalFileBackend(path)); + } + case "default": + return new DelegatingStorage(new InMemoryStorage()); + default: + throw new ArgumentOutOfRangeException(nameof(kind), kind, "Unknown storage kind."); + } + } + + // ------------------------------------------------------------------ same result as single writes + + /// + /// For trial user and system attributes and study user and system attributes, a batch over attributes that + /// partly overwrite earlier ones leaves the same keys, in the same order, with the same values, as one single + /// write per entry. + /// + /// The storage kind. + [TestMethod] + [DataRow("memory")] + [DataRow("journal")] + [DataRow("default")] + public void ABatch_LeavesWhatOneSingleWritePerEntryLeaves(string kind) + { + BaseStorage bulk = Open(kind), single = Open(kind); + int bulkStudy = bulk.CreateNewStudy(Minimize, "s"), singleStudy = single.CreateNewStudy(Minimize, "s"); + int bulkTrial = bulk.CreateNewTrial(bulkStudy), singleTrial = single.CreateNewTrial(singleStudy); + foreach (BaseStorage storage in new[] { bulk, single }) + { + int trialId = storage == bulk ? bulkTrial : singleTrial; + int studyId = storage == bulk ? bulkStudy : singleStudy; + storage.SetTrialUserAttr(trialId, "name", "earlier"); + storage.SetTrialSystemAttr(trialId, "seed", 1L); + storage.SetStudyUserAttr(studyId, "flag", false); + storage.SetStudySystemAttr(studyId, "ratio", 9.0); + } + + OrderedDict attrs = SampleAttrs(); + bulk.SetTrialUserAttrs(bulkTrial, attrs); + bulk.SetTrialSystemAttrs(bulkTrial, attrs); + bulk.SetStudyUserAttrs(bulkStudy, attrs); + bulk.SetStudySystemAttrs(bulkStudy, attrs); + foreach (var (key, value) in attrs) + { + single.SetTrialUserAttr(singleTrial, key, value); + single.SetTrialSystemAttr(singleTrial, key, value); + single.SetStudyUserAttr(singleStudy, key, value); + single.SetStudySystemAttr(singleStudy, key, value); + } + + Describe(bulk.GetTrial(bulkTrial).UserAttrs).Should().Be(Describe(single.GetTrial(singleTrial).UserAttrs)); + Describe(bulk.GetTrial(bulkTrial).SystemAttrs).Should().Be(Describe(single.GetTrial(singleTrial).SystemAttrs)); + Describe(bulk.GetStudyUserAttrs(bulkStudy)).Should().Be(Describe(single.GetStudyUserAttrs(singleStudy))); + Describe(bulk.GetStudySystemAttrs(bulkStudy)).Should().Be(Describe(single.GetStudySystemAttrs(singleStudy))); + bulk.GetTrial(bulkTrial).UserAttrs.Keys.Should().Equal( + "name", "seed", "ratio", "flag", "missing", "list", "nested"); + } + + /// + /// The Optuna-name overloads take the same path: SetUserAttr(IReadOnlyDictionary) on a trial and a study, + /// and SetTrialUserAttr(int, IReadOnlyDictionary) and its three siblings on a storage. + /// + [TestMethod] + public void TheOptunaNameOverloads_WriteTheBatch() + { + var storage = new InMemoryStorage(); + Study study = Optuna.CreateStudy(storage: storage); + Trial trial = study.Ask(); + var attrs = new Dictionary { ["a"] = 1L, ["b"] = "two" }; + + trial.SetUserAttr(attrs); + study.SetUserAttr(attrs); + storage.SetTrialSystemAttr(trial.TrialId, attrs); + storage.SetStudySystemAttr(study.StudyId, attrs); + + trial.UserAttrs.Should().Contain(attrs); + study.UserAttrs.Should().Contain(attrs); + storage.GetTrial(trial.TrialId).UserAttrs.Should().Contain(attrs); + storage.GetTrial(trial.TrialId).SystemAttrs.Should().Contain(attrs); + storage.GetStudySystemAttrs(study.StudyId).Should().Contain(attrs); + + storage.SetTrialUserAttr(trial.TrialId, new Dictionary { ["c"] = 3L }); + storage.SetStudyUserAttr(study.StudyId, new Dictionary { ["d"] = 4L }); + storage.GetTrial(trial.TrialId).UserAttrs["c"].Should().Be(3L); + storage.GetStudyUserAttrs(study.StudyId)["d"].Should().Be(4L); + } + + /// + /// updates the trial's own view at once, and a and a + /// take the per-entry default; the Optuna-name overload works on every trial kind. + /// + [TestMethod] + public void EveryTrialKind_TakesABatch() + { + Study study = Optuna.CreateStudy(); + Trial trial = study.Ask(); + trial.SetUserAttrs(SampleAttrs()); + Describe(trial.UserAttrs).Should().Be(Describe(SampleAttrs())); + study.Tell(trial, 1.0); + Describe(study.Trials[0].UserAttrs).Should().Be(Describe(SampleAttrs())); + + FrozenTrial frozen = FrozenTrial.CreateTrial(state: TrialState.Complete, value: 1.0); + frozen.SetUserAttr(new Dictionary { ["x"] = 1L }); + frozen.UserAttrs["x"].Should().Be(1L); + + var fixedTrial = new FixedTrial(new OrderedDict()); + fixedTrial.SetUserAttrs(new Dictionary { ["y"] = 2L }); + fixedTrial.UserAttrs["y"].Should().Be(2L); + } + + // ------------------------------------------------------------------ nothing written on refusal + + /// + /// A finished trial refuses a batch with the single write's exception and message, and gets nothing written; on + /// a journal not even a record is appended. An unknown trial and an unknown study refuse with a key error. + /// + /// The storage kind. + [TestMethod] + [DataRow("memory")] + [DataRow("journal")] + public void AFinishedOrUnknownTarget_RefusesTheBatch_AndGetsNothingWritten(string kind) + { + BaseStorage storage = Open(kind); + int studyId = storage.CreateNewStudy(Minimize, "refuse"); + int trialId = storage.CreateNewTrial(studyId); + storage.SetTrialUserAttr(trialId, "kept", 1L); + storage.SetTrialStateValues(trialId, TrialState.Complete, new[] { 1.0 }); + long linesBefore = JournalLines(storage); + + Action single = () => storage.SetTrialUserAttr(trialId, "a", 1L); + string message = single.Should().Throw().Which.Message; + long linesAfterSingle = JournalLines(storage); + + Action batch = () => storage.SetTrialUserAttrs(trialId, SampleAttrs()); + Action systemBatch = () => storage.SetTrialSystemAttrs(trialId, SampleAttrs()); + batch.Should().Throw().WithMessage(message); + systemBatch.Should().Throw().WithMessage(message); + storage.GetTrial(trialId).UserAttrs.Keys.Should().Equal("kept"); + JournalLines(storage).Should().Be(linesAfterSingle, "a refused batch appends no record"); + _ = linesBefore; + + Action unknownTrial = () => storage.SetTrialUserAttrs(424242, SampleAttrs()); + Action unknownStudy = () => storage.SetStudyUserAttrs(424242, SampleAttrs()); + unknownTrial.Should().Throw(); + unknownStudy.Should().Throw(); + JournalLines(storage).Should().Be(linesAfterSingle); + } + + /// An empty batch writes and checks nothing; a null one is refused before anything happens. + /// The storage kind. + [TestMethod] + [DataRow("memory")] + [DataRow("journal")] + [DataRow("default")] + public void AnEmptyBatch_DoesNothing_AndANullOneIsRefused(string kind) + { + BaseStorage storage = Open(kind); + int studyId = storage.CreateNewStudy(Minimize, "empty"); + int trialId = storage.CreateNewTrial(studyId); + long lines = JournalLines(storage); + + storage.SetTrialUserAttrs(trialId, new Dictionary()); + storage.SetStudyUserAttrs(studyId, new Dictionary()); + storage.SetTrialUserAttrs(424242, new Dictionary()); // nothing to write, nothing checked + JournalLines(storage).Should().Be(lines); + + Action nullTrial = () => storage.SetTrialUserAttrs(trialId, null!); + Action nullStudy = () => storage.SetStudySystemAttrs(studyId, null!); + nullTrial.Should().Throw(); + nullStudy.Should().Throw(); + } + + // ------------------------------------------------------------------ one step + + /// + /// In memory, a batch of ten attributes on a trial a reader holds copies the record once, then writes every entry + /// into the copy; the reader's record keeps its contents. + /// + [TestMethod] + public void InMemory_ABatchCopiesAHeldRecordOnce() + { + var storage = new InMemoryStorage(); + int studyId = storage.CreateNewStudy(Minimize, "copy"); + int trialId = storage.CreateNewTrial(studyId); + FrozenTrial held = storage.GetTrial(trialId); + var attrs = Enumerable.Range(0, 10).ToDictionary(i => $"k{i}", i => (object?)(long)i); + + storage.SetTrialUserAttrs(trialId, attrs); + + storage.WriteCopyCount.Should().Be(1); + held.UserAttrs.Should().BeEmpty(); + storage.GetTrial(trialId).UserAttrs.Should().HaveCount(10); + } + + /// + /// On a journal, a batch is one record per entry, each the single-key record a single write appends (Python's + /// reader asserts one key per record), all appended in one call; another storage replaying the file sees the same + /// attributes. + /// + [TestMethod] + public void Journal_ABatchIsOrdinarySingleKeyRecords_AppendedTogether() + { + var storage = (JournalStorage)Open("journal"); + string path = _paths[^1]; + int studyId = storage.CreateNewStudy(Minimize, "records"); + int trialId = storage.CreateNewTrial(studyId); + string[] before = File.ReadAllLines(path); + + storage.SetTrialUserAttrs(trialId, SampleAttrs()); + + string[] added = File.ReadAllLines(path).Skip(before.Length).ToArray(); + added.Should().HaveCount(SampleAttrs().Count); + var keys = new List(); + foreach (string line in added) + { + var record = (OrderedDict)PyJson.Loads(line)!; + Convert.ToInt32(record["op_code"]).Should().Be((int)JournalOperation.SetTrialUserAttr); + Convert.ToInt32(record["trial_id"]).Should().Be(trialId); + var userAttr = (OrderedDict)record["user_attr"]!; + userAttr.Should().HaveCount(1, "Python's replay asserts len(log['user_attr']) == 1"); + keys.Add(userAttr.Keys.Single()); + } + keys.Should().Equal(SampleAttrs().Keys); + + var replay = new JournalStorage(new JournalFileBackend(path)); + Describe(replay.GetTrial(trialId).UserAttrs).Should().Be(Describe(SampleAttrs())); + } + + /// + /// If another process finishes the trial between the batch's check and its append, the replay rejects every record + /// of the batch, one per synchronization. The batch write consumes them all and raises once; later, unrelated + /// calls of the storage do not raise for the batch's leftovers. + /// + [TestMethod] + public void Journal_ABatchRejectedAfterItsCheck_IsDrained_SoLaterCallsDoNotRaise() + { + var writer = (JournalStorage)Open("journal"); + string path = _paths[^1]; + int studyId = writer.CreateNewStudy(Minimize, "drain"); + int trialId = writer.CreateNewTrial(studyId); + int otherTrial = writer.CreateNewTrial(studyId); + + // Another worker finishes the trial; the writer has not synchronized since. + var otherProcess = new JournalStorage(new JournalFileBackend(path)); + otherProcess.SetTrialStateValues(trialId, TrialState.Fail); + + // The batch as WriteAttributeBatch builds it after a check that passed on the writer's stale state. + string workerId = writer.WorkerIdForTests; + var logs = SampleAttrs().Select(kv => new OrderedDict + { + ["op_code"] = (int)JournalOperation.SetTrialUserAttr, + ["worker_id"] = workerId, + ["trial_id"] = trialId, + ["user_attr"] = new OrderedDict { [kv.Key] = kv.Value }, + }).ToList(); + + Action batch = () => writer.AppendBatchAndSyncForTests(logs); + batch.Should().Throw(); + + // Every leftover was consumed: these would raise for the batch's remaining records otherwise. + Action read = () => writer.GetTrial(otherTrial); + read.Should().NotThrow(); + writer.SetTrialUserAttrs(otherTrial, new Dictionary { ["fine"] = 1L }); + writer.GetTrial(otherTrial).UserAttrs["fine"].Should().Be(1L); + writer.GetTrial(trialId).UserAttrs.Should().BeEmpty(); + } + + /// + /// The base implementation, which storages without an override (the gRPC client, third-party storages) use, makes + /// exactly one single-entry call per entry, in the dictionary's order. + /// + [TestMethod] + public void TheBaseImplementation_CallsTheSingleWriteOncePerEntry_InOrder() + { + var storage = new DelegatingStorage(new InMemoryStorage()); + int studyId = storage.CreateNewStudy(Minimize, "default"); + int trialId = storage.CreateNewTrial(studyId); + + storage.SetTrialUserAttrs(trialId, SampleAttrs()); + storage.SetStudySystemAttrs(studyId, new Dictionary { ["p"] = 1L, ["q"] = 2L }); + + storage.Calls.Should().Equal( + SampleAttrs().Keys.Select(k => $"trial-user:{k}").Concat(new[] { "study-system:p", "study-system:q" })); + } + + // ------------------------------------------------------------------ helpers + + /// Describes attributes as JSON in their order, so dictionaries of nested values compare by content. + /// The attributes. + /// The description. + private static string Describe(IReadOnlyDictionary attrs) + => string.Join("; ", attrs.Select(kv => $"{kv.Key}={PyJson.Dumps(kv.Value)}")); + + /// The number of lines of a journal storage's file, or -1 for other storages. + /// The storage. + /// The line count. + private long JournalLines(BaseStorage storage) + => storage is JournalStorage ? File.ReadAllLines(_paths[^1]).LongLength : -1; + + /// + /// A storage that forwards every member to an in-memory storage and overrides none of the bulk writes, so they run + /// 's per-entry default; it records the single attribute writes it receives. + /// + private sealed class DelegatingStorage : BaseStorage + { + private readonly InMemoryStorage _inner; + + /// Wraps . + /// The storage every call goes to. + internal DelegatingStorage(InMemoryStorage inner) => _inner = inner; + + /// The single attribute writes received, as kind:key. + internal List Calls { get; } = new(); + + /// + public override int CreateNewStudy(IReadOnlyList directions, string? studyName = null) + => _inner.CreateNewStudy(directions, studyName); + + /// + public override void DeleteStudy(int studyId) => _inner.DeleteStudy(studyId); + + /// + public override void SetStudyUserAttr(int studyId, string key, object? value) + { + Calls.Add($"study-user:{key}"); + _inner.SetStudyUserAttr(studyId, key, value); + } + + /// + public override void SetStudySystemAttr(int studyId, string key, object? value) + { + Calls.Add($"study-system:{key}"); + _inner.SetStudySystemAttr(studyId, key, value); + } + + /// + public override int GetStudyIdFromName(string studyName) => _inner.GetStudyIdFromName(studyName); + + /// + public override string GetStudyNameFromId(int studyId) => _inner.GetStudyNameFromId(studyId); + + /// + public override List GetStudyDirections(int studyId) => _inner.GetStudyDirections(studyId); + + /// + public override OrderedDict GetStudyUserAttrs(int studyId) => _inner.GetStudyUserAttrs(studyId); + + /// + public override OrderedDict GetStudySystemAttrs(int studyId) => _inner.GetStudySystemAttrs(studyId); + + /// + public override List GetAllStudies() => _inner.GetAllStudies(); + + /// + public override int CreateNewTrial(int studyId, FrozenTrial? templateTrial = null) + => _inner.CreateNewTrial(studyId, templateTrial); + + /// + public override void SetTrialParam(int trialId, string paramName, double paramValueInternal, BaseDistribution distribution) + => _inner.SetTrialParam(trialId, paramName, paramValueInternal, distribution); + + /// + public override bool SetTrialStateValues(int trialId, TrialState state, IReadOnlyList? values = null) + => _inner.SetTrialStateValues(trialId, state, values); + + /// + public override void SetTrialIntermediateValue(int trialId, long step, double intermediateValue) + => _inner.SetTrialIntermediateValue(trialId, step, intermediateValue); + + /// + public override void SetTrialUserAttr(int trialId, string key, object? value) + { + Calls.Add($"trial-user:{key}"); + _inner.SetTrialUserAttr(trialId, key, value); + } + + /// + public override void SetTrialSystemAttr(int trialId, string key, object? value) + { + Calls.Add($"trial-system:{key}"); + _inner.SetTrialSystemAttr(trialId, key, value); + } + + /// + public override FrozenTrial GetTrial(int trialId) => _inner.GetTrial(trialId); + + /// + public override List GetAllTrials(int studyId, bool deepcopy = true, IReadOnlyCollection? states = null) + => _inner.GetAllTrials(studyId, deepcopy, states); + } +} From f0d62a0610863e47ee10b26feab298f233f1642f Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 09:11:27 +0200 Subject: [PATCH 016/145] feat(storage): opt-in compact finished trials in InMemoryStorage (plan 13, item S12); same trials, a fifth of the memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner, 2026-11-09: S12 "Build it", "If it changes optuna's/rustuna's behavior then make the feature optin". It changes no result (seeded studies give the same trials, bit for bit, and live-Optuna parity holds on it), but finished trials stop being dictionaries: their read-only accessors return views, numbers and long strings are rebuilt per read, and a finished record no longer shares its dictionaries with the running record it was copied from. So it is a switch, off by default: new InMemoryStorage(compactFinishedTrials: true) // also: storage.CompactFinishedTrials What a finished trial becomes (src/OptunaSharp.Core/Internal/CompactTrialParts.cs, new): - Parameters: CompactParams = one internal-representation double per name against a ParamSchema (names + distribution instances) shared by the study's trials through a per-study ParamSchemaCache (at most 256 shapes; further shapes keep their dictionaries). A trial is accepted only when every stored value comes back indistinguishable from d.ToExternalRepr(d.ToInternalRepr(stored)) (DistributionIdentity.IsSameValue: same type and bits for numbers, -0.0 stays -0.0; same characters for strings; the same object otherwise), so reads return what was stored. Fallbacks that keep the dictionary, each pinned by a test: an int under a float distribution (Optuna keeps it an int), an integer beyond 2^53, a categorical choice Python-equal to an earlier one (1 then 1.0), a value its distribution cannot convert, names listed in another order than the distributions. - Distributions: the schema's one read-only DistributionsView, shared by every trial of the schema. Schemas match distributions by DistributionIdentity.AreInterchangeable: bitwise for float/int (S7's rule), identical choices for a plain CategoricalDistribution (every SuggestCategorical call builds a new distribution, so identity matching would give every categorical trial its own schema), the same instance for anything else. - Attributes: CompactAttrs = key and value arrays (one shared Empty), long ASCII strings as CompactString bytes: ASCII one byte a char, lowercase hex two digits a byte with an odd trailing digit kept aside (CMA-ES splits its pickle hex into 2,045-char chunks), uppercase hex and non-ASCII stay strings. Nested containers stay the same objects. - Intermediate values: an empty dictionary becomes a shared sentinel never handed out; a non-empty one a copy sized to its entries. FrozenTrial (src/OptunaSharp.Core/FrozenTrial.cs): - The four dictionary fields are typed IReadOnlyDictionary and hold either the OrderedDict or a compact form; Params/Distributions/UserAttrs/SystemAttrs return the field as is (unchanged objects on the default path). - ParamsDict/DistributionsDict/UserAttrsDict/SystemAttrsDict and IntermediateValues return real dictionaries: on a compact record the first read materializes one and installs it with Interlocked.CompareExchange (Decompact), so however many threads ask at once they all get the one dictionary the record keeps, and a write through it stays. - New internal: CompactFinished(schemas), IsCompact, IntermediateValuesForRead (the non-pinning read). - Equals, ToString, DeepCopy, LastStep, Constraints, Validate read through the views (comparing and printing never decompact); SetUserAttr, Suggest's distribution write and SetConstraint write through the mutable accessors; ShallowCopy gives a compact record its own dictionaries first and shares them, as Python's copy.copy shares dicts. InMemoryStorage: the switch; compaction in SetTrialStateValues on the new record before it is published (under the lock, only reading the dictionaries the old running record may still share) and in CreateNewTrial for a template that is already finished (Study.AddTrial). GetTrialParam returns a compact trial's stored internal value directly. Read paths kept compact: about 70 internal reads of finished trials moved from the mutable accessors to the read-only ones (no change without the switch: both return the same object): CMA-ES/DE/ES/LM-CMA/VkD-CMA/PSO/JANUS Transform(t.Params); GP, SAASBO, TuRBO, the terminators; TPE's GetParams (now IReadOnlyDictionary); NSGA-II/MOEA-D/ NSGA-III-CPT crossovers and conditional params; QMC, intersection and group-decomposed search spaces (SearchSpaceGroup.AddDistributions takes IReadOnlyDictionary); BaseGASampler generations; SuccessiveHalving, Percentile and Wilcoxon pruners; importance evaluators; dataframe; retry callbacks; Study.ShouldSkipEnqueue; visualization; the CLI records; the BoTorch and pymoo bridges. TPE fast path: GetInternalRepr copies a compact trial's stored internal value instead of boxing and converting the external one, for parameters whose search-space distribution is interchangeable with the trial's (ParamSchema.PositionsFor). Exact by construction: compaction stored internal = d.ToInternalRepr(stored) only when d.ToExternalRepr(internal) is indistinguishable from stored, and an interchangeable distribution converts as d does. Tests: - test/OptunaSharp.Tests/Storages/CompactFinishedTrialsTests.cs (new, 35 cases): every kind of value reads back identical (types, bits, order, reprs, constraints, LastStep, GetTrialParam); Equals/ToString match Optuna's representation both ways; views read like dictionaries (generic and non-generic, CopyTo, KeyNotFound message) and refuse writes; mutable accessors keep writes and stay single under 8-thread contention; the five fallbacks; schema sharing per shape incl. per-trial categorical distributions and the 256 cap; identity rules; compact strings; 15 sampler families run seeded on and off to identical trials with every finished record still compact; CMA-ES chunks stored as hex bytes and restored by a second sampler; 4 workers + 3 readers; retained memory below a quarter. Mutation checks: TPE reading through ParamsDict fails exactly the 5 TPE rows; a dropped trailing hex digit fails 19 cases; a plain assignment instead of the compare-exchange fails the contention test 3 runs of 3. - test/OptunaSharp.Tests.Interop/CompactStorageParityTests.cs (new, 9 cases, live Optuna 5.0): Random with long ASCII, odd hex, non-ASCII and nested attributes, NSGA-II mixed space, QMC and CMA-ES bit-identical full state (CMA-ES chunk strings equal to an uncompacted C# run's), TPE multivariate within the erf tolerance; records stay compact. Proof (scratchpad proof-s12, the plan 13 workloads: 40 floats, seed 0, plain then compact back to back, every pair hash-identical; retained = heap the finished study keeps after a full collection): workload retained plain -> compact peak WS time (steady) allocated random 10k 64.2 -> 6.7 MB (6732->704 B) 200 -> 91 MB 3.52 -> 3.53 s 2.31 -> 2.32 GB qmc 10k 64.2 -> 6.7 MB 198 -> 93 MB 1.02 -> 0.85 s 1.58 -> 1.60 GB nsga2 10k 67.7 -> 9.7 MB (7099->1012 B) 205 -> 103 MB 1.57 -> 1.44 s 1.23 -> 1.29 GB tpe 1k 13.4 -> 6.9 MB 109 -> 92 MB 3.84 -> 3.37 s 1.57 -> 1.60 GB motpe 1k 7.2 -> 1.4 MB 76 -> 77 MB 12.7 -> 11.6 s 3.42 -> 3.45 GB cmaes 10k 129.5 -> 26.5 MB 372 -> 137 MB 2.15 -> 2.46 s * 1.74 -> 1.85 GB nsga2 100k 670.9 -> 87.5 MB (7035->917 B) 751 -> 167 MB 15.7 -> 14.6 s 12.4 -> 13.0 GB cmaes 100k 1287.3 -> 256.8 MB 1846 -> 458 MB 23.2 -> 22.1 s 17.4 -> 18.4 GB * repeated alternately 3x3: steady reps average 2.28 s plain vs 2.31 s compact, i.e. noise. Before the two allocation fixes found by gc-verbose profiling (TPE boxing its history read; the constraints scan rebuilding every attribute string), compact TPE allocated 3.06 GB vs 1.57 and MOTPE 3.90 vs 3.42; now equal. Also fixed on the way (same results, both modes): GetConstraintsFromSystemAttrs scans keys and reads only matching values, so a compact trial rebuilds no attribute string it does not need. Gates on the final code: unit CI filter 2105 passed / 1 skipped / 0 failed on net8.0 and net10.0 (2070 before + 35); live-Python interop (net10.0, engine required, minus Slow) 692 passed / 24 skipped / 0 failed; Storage 455 passed / 144 skipped (no Redis/PG/MySQL services) / 0 failed; CLI 91 passed + its 4 known [OpenBugs] QA reproductions; check_documentation.cs 0 issues; Core and the touched satellites rebuilt --no-incremental with the doc gate's WarningsAsErrors: 0 errors. Docs: performance.md (Memory: compact finished trials; Writing several attributes at once), storage.md, README, study-and-trials.md (SetUserAttrs), THREADING.md §6.1, plan 13 status, §10 rows (S8, S12, D2) and new §10.2. --- README.md | 5 + docs/THREADING.md | 11 + docs/plans/13-rustuna-derived-performance.md | 70 +- docs/website-src/docs/performance.md | 40 + docs/website-src/docs/storage.md | 5 + docs/website-src/docs/study-and-trials.md | 4 + src/OptunaSharp.Cli/Records.cs | 4 +- src/OptunaSharp.Core/FrozenTrial.cs | 297 +++++- .../Importance/BaseImportanceEvaluator.cs | 4 +- .../PedAnova/PedAnovaImportanceEvaluator.cs | 4 +- .../PedAnova/ScottParzenEstimator.cs | 4 +- .../Internal/CompactTrialParts.cs | 963 +++++++++++++++++ .../Internal/ConstrainedOptimization.cs | 7 +- .../Internal/TpeTrialSplit.cs | 6 +- .../Internal/TrialsDataframe.cs | 8 +- .../Pruners/PercentilePruner.cs | 2 +- .../Pruners/SuccessiveHalvingPruner.cs | 6 +- .../Pruners/WilcoxonPruner.cs | 6 +- .../Samplers/BaseGASampler.cs | 4 +- .../Samplers/BruteForceSampler.cs | 6 +- src/OptunaSharp.Core/Samplers/CmaEsSampler.cs | 10 +- .../Samplers/DifferentialEvolutionSampler.cs | 4 +- .../EvolutionStrategySampler.cs | 2 +- src/OptunaSharp.Core/Samplers/GPSampler.cs | 10 +- .../Samplers/Janus/JanusMooCmix.cs | 8 +- .../Samplers/Janus/JanusOptimizer.cs | 4 +- src/OptunaSharp.Core/Samplers/LmCmaSampler.cs | 8 +- .../Samplers/Moead/MoeadOffspring.cs | 6 +- .../Nsgaii/NSGAIIChildGenerationStrategy.cs | 2 +- .../Samplers/Nsgaii/NsgaiiCrossover.cs | 6 +- .../Samplers/NsgaiiiCpt/ConditionalParams.cs | 8 +- .../NsgaiiiCptChildGenerationStrategy.cs | 2 +- .../NsgaiiiCpt/NsgaiiiCptCrossover.cs | 10 +- .../NsgaiiiCpt/NsgaiiiCptSearchSpace.cs | 6 +- src/OptunaSharp.Core/Samplers/PsoSampler.cs | 4 +- src/OptunaSharp.Core/Samplers/QMCSampler.cs | 4 +- .../Samplers/RandomEmbeddingSampler.cs | 2 +- .../Samplers/SaasboSampler.cs | 2 +- src/OptunaSharp.Core/Samplers/TPESampler.cs | 43 +- src/OptunaSharp.Core/Samplers/TurboSampler.cs | 4 +- .../Samplers/VkdCmaSampler.cs | 8 +- .../SearchSpace/GroupDecomposedSearchSpace.cs | 9 +- .../SearchSpace/IntersectionSearchSpace.cs | 4 +- src/OptunaSharp.Core/Storages/Callbacks.cs | 6 +- .../Storages/InMemoryStorage.cs | 73 +- src/OptunaSharp.Core/Study.cs | 6 +- .../Terminator/EmmrEvaluator.cs | 2 +- .../Terminator/ImprovementEvaluator.cs | 2 +- .../Visualization/IntermediateValues.cs | 4 +- .../Visualization/VisualizationUtils.cs | 3 +- .../BoTorchSampler.cs | 4 +- src/OptunaSharp.Interop.Pymoo/PymooSampler.cs | 2 +- .../CompactStorageParityTests.cs | 256 +++++ .../Storages/CompactFinishedTrialsTests.cs | 973 ++++++++++++++++++ 54 files changed, 2792 insertions(+), 161 deletions(-) create mode 100644 src/OptunaSharp.Core/Internal/CompactTrialParts.cs create mode 100644 test/OptunaSharp.Tests.Interop/CompactStorageParityTests.cs create mode 100644 test/OptunaSharp.Tests/Storages/CompactFinishedTrialsTests.cs diff --git a/README.md b/README.md index 8275a8e..346eede 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,11 @@ CMA-ES pickles its state into the storage every generation, and TPE scores candi [Performance and Compatibility](docs/website-src/docs/performance.md) guide lists the opt-in switches that trade that compatibility for speed. +For memory, `new InMemoryStorage(compactFinishedTrials: true)` keeps finished trials compactly and gives the same +trials: the 100,000-trial CMA-ES study above keeps 257 MB instead of 1,287 MB once finished, and NSGA-II's 88 MB +instead of 671 MB, in the same time. Finished trials then hand out read-only views instead of dictionaries; the +guide lists what that changes. + ### Executors Trials run on [OptunaSharp's own dedicated threads by default, or on the .NET ThreadPool][parallel-executor] diff --git a/docs/THREADING.md b/docs/THREADING.md index 8cf3549..b506440 100644 --- a/docs/THREADING.md +++ b/docs/THREADING.md @@ -575,6 +575,17 @@ and ends, around one native call to OpenBLAS). first, as Optuna's copy-on-write does on every write. A finished record is never written again, so the view reads it without the lock. Records read that way are the storage's own, shared by every thread: never mutate them (§10). +- With `InMemoryStorage(compactFinishedTrials: true)` (plan 13, item S12) a finished record holds compact, + immutable forms instead of dictionaries. The storage builds them under its lock, on the new record object + before it publishes it, and only reads the dictionaries it compacts (the running record a reader may hold + shares them), so no reader ever sees a record change form. The read-only accessors return the compact + form itself, which any number of threads may read. The mutable accessors (`ParamsDict` and the other + `…Dict` properties, `IntermediateValues`) give the record back a real dictionary with a compare-exchange: + however many threads ask at once, one dictionary becomes the record's and every caller gets that one, so a + write through any of them stays. A thread that loaded the compact form a moment earlier keeps reading it, + which is safe because it holds the same entries. Pinned by + `CompactFinishedTrialsTests.MutableAccessors_UnderContention_AllReturnTheOneDictionary` and + `ParallelWorkers_AndReaders_SeeConsistentRecords`. ### 6.2 Trial diff --git a/docs/plans/13-rustuna-derived-performance.md b/docs/plans/13-rustuna-derived-performance.md index 18686b7..2af81f1 100644 --- a/docs/plans/13-rustuna-derived-performance.md +++ b/docs/plans/13-rustuna-derived-performance.md @@ -1,9 +1,10 @@ # Plan 13 — Rustuna-derived performance: compatible speedups first, opt-in paths where results would change -> **Status: BUILT (2026-11-09), except S8 and S12 (§10).** Phases 1-4 are on `main`: `8f8ce3e` (S1-S3, D1), -> `605806c` (S4, S7), `dce5885` (S3, S5, S6), `656ab94` (S10, S9 as measured). S11 was measured and changes nothing; -> D2 awaits the owner with both of its numbers. §10 records what was built, how it was gated, and the before/after -> measurements. Background, benchmarks and the full Rustuna read-through: catalog note +> **Status: BUILT (2026-11-09).** Phases 1-4 are on `main`: `8f8ce3e` (S1-S3, D1), +> `605806c` (S4, S7), `dce5885` (S3, S5, S6), `656ab94` (S10, S9 as measured). The owner then decided the three open +> items (2026-11-09): S8 built (`74893f7`), S12 built as an opt-in (`InMemoryStorage(compactFinishedTrials: true)`, +> §10.2), D2 not built (exact kernel only). S11 was measured and changes nothing. §10 records what was built, how it +> was gated, and the before/after measurements. Background, benchmarks and the full Rustuna read-through: catalog note > `optunasharp/notes/rustuna-comparison-2026-11-08.md`. --- @@ -368,9 +369,9 @@ the `de495f8` snapshot and the working tree; and, for the concurrency-sensitive | S10 | `656ab94` | `CachedStorage` keeps its cache in number order (`ByNumber`, placeholders for unsynced numbers); `GetAllTrials` stops sorting and `GetTrialsPrefixView` wraps the cache | O(n log n) → O(n) per read, O(1) view | | S9 | `656ab94` | Measured first: attribute JSON is 3-4 % of a 10k-trial SQLite cold load, so lazy decoding was not built. Parsing each `distribution_json` row was ~40 %: both RDB storages now parse each distinct text once per load (`JournalDistributionCache`) | Cold load 2.1 → 1.4-1.5 s, 1.22 GB → 479 MB | | S11 | none | Measured: at 10-D and 40-D CMA-ES and 6-D GP the OpenBLAS thread count changes no result, but at 100-D every count gives a different (deterministic) result, as NumPy's does. Pinning one thread at `nJobs: 1` would break parity with Python's default-thread NumPy, so the automatic policy stays (pin only when `nJobs > 1`). One thread was 12 % faster at 40-D and 24-35 % at 100-D: available through `TrialExecutorOptions.BlasThreads = 1`, at the cost of parity | — | -| S8 | not built | The bulk attribute API adds public surface Optuna lacks; after S4 an in-memory attribute write is already in place (O(1)), and the measured database cost is dominated by per-suggestion parameter writes, which no attribute API can batch. Left for the owner | — | -| S12 | not built | Measured instead: S4 and S7 took the 100k-trial CMA-ES study from ~2.5 GB to 1.44 GB peak working set. The rest is per-trial dictionaries and boxed values plus the generation pickles (UTF-16 strings); compacting them means materializing `FrozenTrial` contents on read, a larger change with its own risks. Left for the owner with these numbers (and D4 with it) | — | -| D2 | pending | Scoring still takes 41 % of a 1,000-trial TPE run and 49 % of a 1,000-trial MOTPE run after S6 (profiles, 2026-11-09). A fast kernel could at best roughly halve that share; §2.4's noise harness bounds the decision risk. Asked of the owner with these numbers | — | +| S8 | `74893f7` | Owner 2026-11-09: "Build it", "Also have optuna's api overload". `SetTrialUserAttrs`/`SetTrialSystemAttrs`/`SetStudyUserAttrs`/`SetStudySystemAttrs` (virtual, default = one call per entry) plus Optuna-named overloads taking a dictionary; `Trial`/`Study.SetUserAttrs` and `SetUserAttr(dict)`. Overrides write in one step: one lock in memory, one transaction in SQLite and EF Core, one appended batch of single-key records in a journal (Python's replay asserts one key per record), with a drain so a rejected own record cannot poison the next call. Same result as the loop; a finished or unknown trial gets nothing written. Live Python reads a C#-written batch | — | +| S12 | this commit | Owner 2026-11-09: "Build it", opt-in if behavior changes. Built as `InMemoryStorage(compactFinishedTrials: true)`: results unchanged (every proof pair hash-identical, live-Optuna parity on it), but finished trials' read-only accessors return views and values are rebuilt per read, hence the switch. §10.2 | CMA-ES 100k: retained 1,287 → 257 MB, peak 1,845 → 460 MB | +| D2 | not built | Owner 2026-11-09: "Keep exact only". TPE scores in NumPy's order with the S6 exact kernel; no fast-numerics path | — | **Findings along the way.** - The CMA-ES pickle stores only C's upper triangle; keeping the told optimizer instead of a restore is therefore @@ -414,3 +415,58 @@ The README's sampler table (benchmark 14's C# probes, rebuilt against `656ab94`, where the change is beyond the busy machine's noise: random over 1,000 parameters 13.5 → 3.2 s (now faster than Python's 8.4 s), CMA-ES 200-D 2.8 → 1.2 s, CMA-ES 6-D 1.75 → 0.91 s, QMC 0.64 → 0.39 s, random 6-D 1.34 → 0.98 s, TPE 100-D 2.4 → 2.1 s. GP was rerun interleaved against a `de495f8` build: the same within noise. + +### 10.2 S12: compact finished trials (opt-in) + +**What it is.** `new InMemoryStorage(compactFinishedTrials: true)` (default false) compacts each record as it finishes +(`Internal/CompactTrialParts.cs`, `FrozenTrial.CompactFinished`): + +- parameters: one internal value per name against a `ParamSchema` the study's trials share; +- distributions: the schema's one shared view; +- attributes: arrays, with long ASCII strings as bytes. Lowercase hexadecimal is stored two digits a byte, an odd + trailing digit kept aside, because CMA-ES's 2,045-character pickle chunks have an odd length. +- empty dictionaries: no memory. + +**Why it keeps results.** A trial is compacted only when every value reads back indistinguishable from what it stored; +the fallbacks keep the dictionary. + +**Why it is a switch.** The read-only accessors return views, values are rebuilt per read, and the mutable accessors +re-materialize the record's dictionary with a compare-exchange. The owner's rule was "if it changes optuna's/rustuna's +behavior then make the feature optin". + +**Read paths.** About 70 internal reads of finished trials moved to the read-only accessors (identical objects without +the switch), so samplers never undo compaction. A test runs every sampler family on and off and asserts every record +is still compact. + +**Two allocation sites found by profiling** (`dotnet-trace --profile gc-verbose`, TraceEvent tally by type and frame): + +- TPE's history read boxed every value per ask. It now copies the stored internal value where the search-space + distribution is interchangeable (`ParamSchema.PositionsFor`). +- `GetConstraintsFromSystemAttrs` walked every attribute pair, rebuilding each trial's relative-parameter JSON string + per ask (86 MB in a 300-trial run). It now scans keys and fetches only matching values. + +With both, compact TPE allocates what plain does. + +**Proof.** The plan 13 workloads (40 floats in [-10, 10], seed 0, sampler defaults) on the final build, plain then +compact back to back in separate processes, 2026-11-09 09:04-09:07, load 20-55 %. Retained = heap the finished study +keeps after a full collection (rep 0). Times are the steady rep where there are two. Every pair hash-identical. + +| Workload | Retained, plain → compact | Per trial | Peak working set | Time | Allocated | +|---|---|---|---|---|---| +| Random, 10,000 | 64.2 → 6.7 MB | 6,732 → 704 B | 200 → 91 MB | 3.52 → 3.53 s | 2.31 → 2.32 GB | +| QMC, 10,000 | 64.2 → 6.7 MB | 6,735 → 704 B | 198 → 93 MB | 1.02 → 0.85 s | 1.58 → 1.60 GB | +| NSGA-II, 10,000 | 67.7 → 9.7 MB | 7,099 → 1,012 B | 205 → 103 MB | 1.57 → 1.44 s | 1.23 → 1.29 GB | +| TPE, 1,000 | 13.4 → 6.9 MB | 14,059 → 7,243 B | 109 → 92 MB | 3.84 → 3.37 s | 1.57 → 1.60 GB | +| MOTPE, 1,000 | 7.2 → 1.4 MB | 7,537 → 1,461 B | 76 → 77 MB | 12.7 → 11.6 s | 3.42 → 3.45 GB | +| CMA-ES, 10,000 | 129.5 → 26.5 MB | 13,581 → 2,779 B | 372 → 137 MB | 2.15 → 2.46 s | 1.74 → 1.85 GB | +| NSGA-II, 100,000 | 670.9 → 87.5 MB | 7,035 → 917 B | 751 → 167 MB | 15.7 → 14.6 s | 12.4 → 13.0 GB | +| CMA-ES, 100,000 | 1,287.3 → 256.8 MB | 13,499 → 2,692 B | 1,846 → 458 MB | 23.2 → 22.1 s | 17.4 → 18.4 GB | + +**Time.** The CMA-ES 10,000 pair was repeated alternately three times with three reps each, and is noise: the steady +reps averaged 2.28 s plain and 2.31 s compact. Earlier runs had it at 2.88 → 2.86 s and 3.41 → 3.09 s. Garbage-collector +pauses drop on the large studies (CMA-ES 100,000: 2.73 → 1.36 s). + +**Before the fixes.** Compact TPE allocated 3.06 GB against plain's 1.57 GB, and MOTPE 3.90 GB against 3.42 GB. + +**Memory per trial.** The probe in session scratchpad `s12probe` agrees: random over 40 parameters 6,681 → 648 B per +trial at 5,000 trials, with the first study warmed up. diff --git a/docs/website-src/docs/performance.md b/docs/website-src/docs/performance.md index c1a56a8..30e9f82 100644 --- a/docs/website-src/docs/performance.md +++ b/docs/website-src/docs/performance.md @@ -48,6 +48,46 @@ With `nJobs` above 1 OptunaSharp already runs OpenBLAS on one thread automatical so they are not reproducible anyway, and several workers in linear algebra at once would otherwise oversubscribe the cores (see [Easy Parallelization](key-features/parallelization.md)). +## Memory: compact finished trials + +`new InMemoryStorage(compactFinishedTrials: true)` stores each trial compactly once it finishes. Its parameters +become one internal value each, stored against a parameter schema that the study's trials share, and its +distributions come from that schema. Its attributes become two arrays, with long ASCII strings stored as bytes: +CMA-ES's optimizer state, a hexadecimal string, takes two digits per byte. Empty dictionaries take no memory. +Results do not change: each pair below gave the same trials (equal hashes of every parameter and value), and the +live-Optuna parity tests pass on it bit for bit. + +| Study (40 parameters, seed 0) | Retained by the finished study | Peak working set | Time | +|---|---|---|---| +| CMA-ES, 100,000 trials | 1,287 MB to 257 MB | 1,846 MB to 458 MB | 23.2 s to 22.1 s | +| NSGA-II, 100,000 trials | 671 MB to 88 MB | 751 MB to 167 MB | 15.7 s to 14.6 s | +| CMA-ES, 10,000 trials | 130 MB to 27 MB | 372 MB to 137 MB | 2.28 s to 2.31 s | +| TPE, 1,000 trials | 13.4 MB to 6.9 MB | 109 MB to 92 MB | 3.8 s to 3.4 s | +| Random, 10,000 trials | 64 MB to 6.7 MB | 200 MB to 91 MB | 3.5 s to 3.5 s | + +The times are the same within the machine's noise: CMA-ES over 10,000 trials averaged 2.28 s against 2.31 s over +six steady runs each, and the large studies spend less time in the garbage collector (CMA-ES over 100,000 trials: +2.7 s of pauses against 1.4 s). Allocation stays within 6 %. It is a switch rather than the default because a +finished trial no longer holds dictionaries: + +- `Params`, `Distributions`, `UserAttrs` and `SystemAttrs` of a finished trial are read-only views, with the same + entries in the same order. A cast to `OrderedDict` fails, and a write through `IDictionary` throws. +- Numbers and long strings are rebuilt on each read, so two reads are equal but are not the same object. +- `ParamsDict`, the other `…Dict` properties and `IntermediateValues` still return real dictionaries. The first such + read gives the trial its dictionary back, and it keeps it, so a change made through it stays; that trial then + loses its saving. +- A finished trial no longer shares its dictionaries with the running record it was copied from. +- Only `InMemoryStorage` has the switch; journal and database storages keep their records as before. + +## Writing several attributes at once + +`trial.SetUserAttrs(attrs)`, `study.SetUserAttrs(attrs)` and the storage methods `SetTrialUserAttrs`, +`SetTrialSystemAttrs`, `SetStudyUserAttrs` and `SetStudySystemAttrs` write a dictionary of attributes in one step. +Optuna's singular names take a dictionary too, as overloads (`trial.SetUserAttr(attrs)`, `storage.SetTrialUserAttr(id, +attrs)`). The result is what one call per entry would leave. The in-memory storage takes its lock once, the journal +appends ordinary one-attribute records in one batch (Python Optuna reads them), and the databases use one +transaction. A finished or unknown trial gets nothing written. + ## Thread safety None of the caches changes what concurrent callers see. A record read from an in-memory storage, through any read, is diff --git a/docs/website-src/docs/storage.md b/docs/website-src/docs/storage.md index 6d8ca60..2227834 100644 --- a/docs/website-src/docs/storage.md +++ b/docs/website-src/docs/storage.md @@ -43,6 +43,11 @@ study.Optimize(trial => }, nTrials: 100); ``` +A long study in memory can keep its finished trials compactly with `new InMemoryStorage(compactFinishedTrials: +true)`: a 100,000-trial CMA-ES study over 40 parameters retains 257 MB instead of 1,288 MB, with the same trials. +Finished trials then return read-only views instead of dictionaries; [Performance and +Compatibility](performance.md#memory-compact-finished-trials) lists what that changes. + ## Journal file storage (persistent, shareable) `JournalStorage` records every change as an appended JSON line; `JournalFileBackend` stores that diff --git a/docs/website-src/docs/study-and-trials.md b/docs/website-src/docs/study-and-trials.md index b74c28d..f85ad7c 100644 --- a/docs/website-src/docs/study-and-trials.md +++ b/docs/website-src/docs/study-and-trials.md @@ -137,8 +137,12 @@ study.Optimize(trial => ## Attributes, enqueueing, and manual trials ```csharp +using OptunaSharp.Collections; + study.SetUserAttr("dataset", "cifar10"); // arbitrary metadata on the study trial.SetUserAttr("fold", 3); // …or on a trial +trial.SetUserAttrs(new OrderedDict // several at once, in one storage write + { ["seed"] = 7, ["split"] = "val" }); study.EnqueueTrial(new() { ["x"] = 0.0 }); // evaluate specific params next study.AddTrial(FrozenTrial.CreateTrial( // inject an already-evaluated trial diff --git a/src/OptunaSharp.Cli/Records.cs b/src/OptunaSharp.Cli/Records.cs index a51fbbe..8ce0236 100644 --- a/src/OptunaSharp.Cli/Records.cs +++ b/src/OptunaSharp.Cli/Records.cs @@ -88,7 +88,7 @@ HashSet Agg() } case "params": { - foreach (var (name, value) in trial.ParamsDict) + foreach (var (name, value) in trial.Params) { var col = new Column(dfColumn, name); record[col] = value; @@ -98,7 +98,7 @@ HashSet Agg() } case "user_attrs": { - foreach (var (name, value) in trial.UserAttrsDict) + foreach (var (name, value) in trial.UserAttrs) { var col = new Column(dfColumn, name); record[col] = value; diff --git a/src/OptunaSharp.Core/FrozenTrial.cs b/src/OptunaSharp.Core/FrozenTrial.cs index 211cd1b..8f53d48 100644 --- a/src/OptunaSharp.Core/FrozenTrial.cs +++ b/src/OptunaSharp.Core/FrozenTrial.cs @@ -1,6 +1,8 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Threading; using NumSharp; using OptunaSharp.Collections; using OptunaSharp.Distributions; @@ -25,10 +27,24 @@ public class FrozenTrial : BaseTrial private int _number; private List? _values; private DateTime? _datetimeStart; - private OrderedDict _params; - private OrderedDict _userAttrs; - private OrderedDict _systemAttrs; - private OrderedDict _distributions; + + // Each dictionary field holds the OrderedDict the trial was built with, or, once a storage built with + // compactFinishedTrials compacted the finished record (Rustuna plan 13, item S12), an immutable compact form + // (ICompactDictionary). The read-only accessors return either as is; the mutable …Dict accessors swap a compact form + // for a real dictionary first (Decompact*), so a write through them is never lost. + private IReadOnlyDictionary _params; + private IReadOnlyDictionary _userAttrs; + private IReadOnlyDictionary _systemAttrs; + private IReadOnlyDictionary _distributions; + private OrderedDict _intermediateValues; + + /// + /// What a compact record holds in place of an empty dictionary (plan 13, item + /// S12): shared by every such record and never handed out, because the public getter returns a mutable dictionary. + /// Internal reads may use it as the empty dictionary it is; replaces it with a + /// record's own dictionary before returning. + /// + private static readonly OrderedDict CompactEmptyIntermediateValues = new(); /// Storage identifier for this trial, separate from its per-study number. internal int TrialId; @@ -100,7 +116,7 @@ public FrozenTrial( _params = @params; _userAttrs = userAttrs; _systemAttrs = systemAttrs; - IntermediateValues = intermediateValues; + _intermediateValues = intermediateValues; _distributions = distributions; TrialId = trialId; } @@ -115,7 +131,31 @@ public FrozenTrial( /// Intermediate objective values set with , keyed by step. Steps are 64-bit, like /// Python's unbounded int steps in journal, SQLite and in-memory storages. /// - public OrderedDict IntermediateValues { get; set; } + /// + /// On a record an built with compactFinishedTrials: true compacted + /// while it had no intermediate values, the first read gives the record its own empty dictionary and keeps it, so + /// the dictionary returned here is always the record's and a change to it stays (plan 13, item S12). + /// + public OrderedDict IntermediateValues + { + get + { + OrderedDict current = _intermediateValues; + if (!ReferenceEquals(current, CompactEmptyIntermediateValues)) + return current; + // Only one dictionary may become the record's: a reader that loses the race returns the winner's. + var own = new OrderedDict(); + OrderedDict witnessed = Interlocked.CompareExchange(ref _intermediateValues, own, current); + return ReferenceEquals(witnessed, current) ? own : witnessed; + } + set => _intermediateValues = value; + } + + /// + /// The intermediate values for internal reads that change nothing: the record's dictionary, or the shared empty + /// one of a compact record, which would replace with a dictionary of its own. + /// + internal OrderedDict IntermediateValuesForRead => _intermediateValues; // ------------------------------------------------------------------ suggest API @@ -192,7 +232,8 @@ public override void Report(double value, long step) /// Attribute name; an existing attribute is overwritten. /// Value retained in this replay record's local attribute dictionary. /// Updates the local user attribute by name. - public override void SetUserAttr(string key, object? value) => _userAttrs[key] = value; + /// Writes through , so a compact record gets a dictionary of its own first. + public override void SetUserAttr(string key, object? value) => UserAttrsDict[key] = value; /// /// Read this trial's observed-components vector (the residual terms written with @@ -282,7 +323,8 @@ internal void Validate() if (_distributions.TryGetValue(name, out var existing)) Distribution.CheckDistributionCompatibility(existing, distribution); - _distributions[name] = distribution; + // Through the mutable accessor: a compact record gets a dictionary of its own before the write. + DistributionsDict[name] = distribution; return value; } @@ -341,32 +383,53 @@ public List? Values public void SetDatetimeStart(DateTime? value) => _datetimeStart = value; /// Dictionary that contains suggested parameters (external representations). + /// + /// On a finished record of an built with compactFinishedTrials: true + /// this is a read-only view over the trial's internal values (plan 13, item S12): the same entries in the same + /// order, each numeric value a new box with the stored bits on every read, and not an + /// . returns a real dictionary in every case. + /// public override IReadOnlyDictionary Params => _params; /// The mutable parameter dictionary (Python's params setter). + /// + /// A compact record (see ) gets a dictionary of its own on the first read, which it keeps: + /// every later read returns that dictionary, and a change to it is the trial's change. It costs the memory the + /// compact form saved, so internal readers that only read use . + /// public OrderedDict ParamsDict { - get => _params; + get => _params as OrderedDict ?? Decompact(ref _params); set => _params = value; } /// Dictionary that contains the distributions of . + /// + /// On a compact record (see ), a read-only view that every trial of the same parameter + /// schema shares; returns a real dictionary in every case. + /// public override IReadOnlyDictionary Distributions => _distributions; /// Gets or replaces the mutable distribution dictionary; entries are shared distribution objects. + /// A compact record gets a dictionary of its own on the first read, as does. public OrderedDict DistributionsDict { - get => _distributions; + get => _distributions as OrderedDict ?? Decompact(ref _distributions); set => _distributions = value; } /// Dictionary that contains the user attributes. + /// + /// On a compact record (see ), a read-only view: long ASCII strings come back as new string + /// instances with the stored characters, every other value as the stored object. + /// public override IReadOnlyDictionary UserAttrs => _userAttrs; /// Gets or replaces the mutable user-attribute dictionary. + /// A compact record gets a dictionary of its own on the first read, as does. public OrderedDict UserAttrsDict { - get => _userAttrs; + get => _userAttrs as OrderedDict ?? Decompact(ref _userAttrs); set => _userAttrs = value; } @@ -375,22 +438,26 @@ public OrderedDict DistributionsDict /// FrozenTrial (only BaseTrial/Trial/Study lost theirs in #6834), so samplers /// read system attributes of finished trials through it. /// - public IReadOnlyDictionary SystemAttrs => SystemAttrsDict; + /// On a compact record, a read-only view as describes. + /// The deferred storage loader fails; the loader remains available for a later attempt. + public IReadOnlyDictionary SystemAttrs + { + get + { + ResolveLazySystemAttrs(); + return _systemAttrs; + } + } /// Gets or replaces internal attributes, resolving any deferred storage lookup once. + /// A compact record gets a dictionary of its own on the first read, as does. /// The deferred storage loader fails; the loader remains available for a later attempt. public OrderedDict SystemAttrsDict { get { - if (_systemAttrsLoader is not null) - { - // Clear the loader only after success, allowing a transient backend failure to be - // retried instead of permanently exposing an obsolete attribute snapshot. - _systemAttrs = _systemAttrsLoader(); - _systemAttrsLoader = null; - } - return _systemAttrs; + ResolveLazySystemAttrs(); + return _systemAttrs as OrderedDict ?? Decompact(ref _systemAttrs); } set { @@ -399,18 +466,135 @@ public OrderedDict DistributionsDict } } + /// Loads deferred system attributes, if any are pending, before either accessor returns them. + /// The deferred storage loader fails; the loader remains available for a later attempt. + private void ResolveLazySystemAttrs() + { + if (_systemAttrsLoader is not null) + { + // Clear the loader only after success, allowing a transient backend failure to be + // retried instead of permanently exposing an obsolete attribute snapshot. + _systemAttrs = _systemAttrsLoader(); + _systemAttrsLoader = null; + } + } + /// Python Trial._get_latest_trial support: defer system-attr loading to the storage. /// Deferred storage read invoked on first SystemAttrsDict access; failures are not swallowed. internal void SetLazySystemAttrs(Func> loader) => _systemAttrsLoader = loader; + // ------------------------------------------------------------------ compact records (plan 13, item S12) + + /// + /// Replaces the compact form in with a real dictionary of the same entries and returns + /// it, so the caller of a mutable accessor gets the dictionary the record keeps. + /// + /// The dictionary's value type. + /// One of the record's dictionary fields, holding a compact form or null. + /// The record's dictionary from now on; null when the field held null (a record built with a null dictionary keeps returning null). + /// + /// Readers on other threads may call this at once on a shared finished record: the compare-exchange lets exactly + /// one materialized dictionary become the record's, and every caller returns that one, so no caller can write into + /// a dictionary the record does not keep. A reader that loaded the compact form a moment earlier keeps reading it, + /// which is safe because a compact form never changes and holds the same entries. + /// + private static OrderedDict Decompact(ref IReadOnlyDictionary field) + { + IReadOnlyDictionary observed = field; + while (true) + { + switch (observed) + { + case null: + return null!; + case OrderedDict dictionary: + return dictionary; + } + OrderedDict own = ((ICompactDictionary)observed).Materialize(); + IReadOnlyDictionary witnessed = Interlocked.CompareExchange(ref field, own, observed); + if (ReferenceEquals(witnessed, observed)) + return own; + // Another thread (or a setter) replaced the form first: return whatever the record holds now. + observed = witnessed; + } + } + + /// + /// True while any of the record's dictionaries is held in compact form (plan 13, item S12); a diagnostic the tests + /// read to prove which records a storage compacted and which a mutable accessor turned back. + /// + internal bool IsCompact + => _params is ICompactDictionary + || _distributions is ICompactDictionary + || _userAttrs is ICompactDictionary + || _systemAttrs is ICompactDictionary + || ReferenceEquals(_intermediateValues, CompactEmptyIntermediateValues); + + /// + /// Stores a finished record compactly (plan 13, item S12): parameters as internal values against a schema shared + /// by the study's trials, attributes as arrays with long ASCII strings as bytes, and nothing for empty + /// dictionaries. Every accessor returns the same entries afterwards. + /// + /// The study's schema cache; the caller (the storage) holds the lock that guards it. + /// + /// + /// The caller must hold the only reference to this record object: the storage calls it on the copy it is about + /// to publish, under its lock, so no reader can see the fields change. The record's dictionaries are only + /// read, never changed, because the record it was copied from (and a reader holding that one) may share + /// them; after this call the published record no longer references them. + /// + /// + /// Parameters stay a dictionary when they cannot be compacted exactly: names in another order than the + /// distributions, more parameter shapes than , or a value that + /// would not read back identical (see ). A record with deferred + /// system attributes keeps them deferred. + /// + /// + internal void CompactFinished(ParamSchemaCache schemas) + { + if (_params is OrderedDict @params + && _distributions is OrderedDict distributions) + { + ParamSchema? schema = schemas.FindOrAdd(@params, distributions); + CompactParams? compact = schema is null ? null : CompactParams.TryCreate(schema, @params); + if (compact is not null) + { + _params = compact; + _distributions = compact.Schema.View; + } + } + + if (_userAttrs is OrderedDict userAttrs) + _userAttrs = CompactAttrs.Create(userAttrs); + if (_systemAttrsLoader is null && _systemAttrs is OrderedDict systemAttrs) + _systemAttrs = CompactAttrs.Create(systemAttrs); + + OrderedDict intermediate = _intermediateValues; + if (intermediate is { Count: 0 }) + { + _intermediateValues = CompactEmptyIntermediateValues; + } + else if (intermediate is not null) + { + // Kept as a dictionary (the public getter returns the mutable type), but as a copy sized to its entries: + // reports grow it by prime steps, and the source may be shared with the record this one was copied from. + var trimmed = new OrderedDict(intermediate.Count); + foreach (var (step, value) in intermediate) + trimmed.Add(step, value); + _intermediateValues = trimmed; + } + } + /// Return the maximum step of in the trial (64-bit, like the steps). public long? LastStep { get { - if (IntermediateValues.Count == 0) + // The read-only view, so a compact record without reports does not get a dictionary just to say so. + OrderedDict intermediateValues = IntermediateValuesForRead; + if (intermediateValues.Count == 0) return null; - return IntermediateValues.Keys.Max(); + return intermediateValues.Keys.Max(); } } @@ -430,7 +614,7 @@ public TimeSpan? Duration /// are zero or less. /// public override OrderedDict Constraints - => ConstrainedOptimization.GetConstraintsFromSystemAttrs(SystemAttrsDict); + => ConstrainedOptimization.GetConstraintsFromSystemAttrs(SystemAttrs); /// /// Set a constraint value for the trial. The trial is considered feasible when all @@ -474,8 +658,9 @@ public override bool Equals(object? obj) && DictValueEquals(_params, other._params) && DistributionsEqual(_distributions, other._distributions) && DictValueEquals(_userAttrs, other._userAttrs) - && DictValueEquals(SystemAttrsDict, other.SystemAttrsDict) - && IntermediateValuesEqual(IntermediateValues, other.IntermediateValues) + // The read-only accessors: comparing must not give a compact record dictionaries of its own. + && DictValueEquals(SystemAttrs, other.SystemAttrs) + && IntermediateValuesEqual(IntermediateValuesForRead, other.IntermediateValuesForRead) && TrialId == other.TrialId; } @@ -519,10 +704,10 @@ private static bool ValuesEqual(List? a, List? b) } /// Compares parameter or attribute dictionaries using Python-compatible nested value equality. - /// First collection to compare. - /// Second collection to compare. + /// First collection to compare; a dictionary or a compact record's read-only view. + /// Second collection to compare; a dictionary or a compact record's read-only view. /// True when both collections contain equal corresponding values. - private static bool DictValueEquals(OrderedDict a, OrderedDict b) + private static bool DictValueEquals(IReadOnlyDictionary a, IReadOnlyDictionary b) { if (a.Count != b.Count) return false; @@ -538,11 +723,11 @@ private static bool DictValueEquals(OrderedDict a, OrderedDict< } /// Compares distribution dictionaries by parameter name and distribution value. - /// First collection to compare. - /// Second collection to compare. + /// First collection to compare; a dictionary or a compact record's read-only view. + /// Second collection to compare; a dictionary or a compact record's read-only view. /// True when both collections contain equal corresponding values. private static bool DistributionsEqual( - OrderedDict a, OrderedDict b) + IReadOnlyDictionary a, IReadOnlyDictionary b) { if (a.Count != b.Count) return false; @@ -572,18 +757,26 @@ private static bool IntermediateValuesEqual(OrderedDict a, Ordered /// Python copy.copy(trial): a shallow copy (dict references shared). /// A new trial wrapper sharing dictionaries, values, and the deferred loader with this instance. + /// + /// A compact record (plan 13, item S12) first gets real dictionaries of its own, which the copy then shares: + /// sharing the compact forms instead would let each object later take a different dictionary, and a change + /// through one would no longer show in the other as it does for Python's shallow copy. The storages copy only + /// unfinished records, which are never compact, so this costs nothing on their paths. + /// internal FrozenTrial ShallowCopy() { + // Field by field rather than through the accessors: SystemAttrsDict would run a pending deferred loader, which + // a shallow copy hands over unrun instead. var copy = new FrozenTrial( number: _number, state: State, value: null, datetimeStart: _datetimeStart, datetimeComplete: DatetimeComplete, - @params: _params, - distributions: _distributions, - userAttrs: _userAttrs, - systemAttrs: _systemAttrs, + @params: _params as OrderedDict ?? Decompact(ref _params), + distributions: _distributions as OrderedDict ?? Decompact(ref _distributions), + userAttrs: _userAttrs as OrderedDict ?? Decompact(ref _userAttrs), + systemAttrs: _systemAttrs as OrderedDict ?? Decompact(ref _systemAttrs), intermediateValues: IntermediateValues, trialId: TrialId, values: null); @@ -594,6 +787,10 @@ internal FrozenTrial ShallowCopy() /// Python copy.deepcopy(trial). /// A new trial with copied value lists and supported nested attribute containers; distribution instances remain shared. + /// + /// The copy always holds real dictionaries, including the copy of a compact record (plan 13, item S12), which is + /// read through its views and left compact itself. + /// public FrozenTrial DeepCopy() { // Nested JSON-compatible values need separate containers because users commonly mutate @@ -605,20 +802,29 @@ public FrozenTrial DeepCopy() datetimeStart: _datetimeStart, datetimeComplete: DatetimeComplete, @params: DeepCopyAttrs(_params), - distributions: _distributions.ShallowClone(), // Distributions are treated as immutable. + distributions: CloneDistributions(_distributions), // Distributions are treated as immutable. userAttrs: DeepCopyAttrs(_userAttrs), - systemAttrs: DeepCopyAttrs(SystemAttrsDict), - intermediateValues: IntermediateValues.ShallowClone(), + systemAttrs: DeepCopyAttrs(SystemAttrs), + intermediateValues: IntermediateValuesForRead.ShallowClone(), trialId: TrialId, values: null); copy._values = _values is null ? null : new List(_values); return copy; } + /// A new distributions dictionary sharing the distribution instances, as ShallowClone builds one. + /// The record's dictionary, or a compact record's shared view. + /// A dictionary the copy owns, in the same order. + private static OrderedDict CloneDistributions( + IReadOnlyDictionary distributions) + => distributions is OrderedDict dictionary + ? dictionary.ShallowClone() + : new OrderedDict(distributions); + /// Copies an attribute dictionary through the Python-compatible nested-value copier. - /// Attribute dictionary to copy. + /// Attribute dictionary to copy; a dictionary or a compact record's read-only view. /// A new dictionary with supported nested containers copied. - private static OrderedDict DeepCopyAttrs(OrderedDict attrs) + private static OrderedDict DeepCopyAttrs(IReadOnlyDictionary attrs) { var copy = new OrderedDict(attrs.Count); foreach (var (key, value) in attrs) @@ -637,11 +843,12 @@ public override string ToString() $"values={(_values is null ? "None" : PythonRepr.ListRepr(_values))}", $"datetime_start={PythonRepr.Repr(_datetimeStart?.ToString("yyyy-MM-dd HH:mm:ss.ffffff"))}", $"datetime_complete={PythonRepr.Repr(DatetimeComplete?.ToString("yyyy-MM-dd HH:mm:ss.ffffff"))}", - $"params={PythonRepr.DictRepr(_params)}", - $"user_attrs={PythonRepr.DictRepr(_userAttrs)}", - $"system_attrs={PythonRepr.DictRepr(_systemAttrs)}", - $"intermediate_values={PythonRepr.DictRepr(IntermediateValues)}", - $"distributions={PythonRepr.DictRepr(_distributions)}", + // Every field is an IDictionary: an OrderedDict, or a compact record's view, which prints the same entries. + $"params={PythonRepr.DictRepr((IDictionary)_params)}", + $"user_attrs={PythonRepr.DictRepr((IDictionary)_userAttrs)}", + $"system_attrs={PythonRepr.DictRepr((IDictionary)_systemAttrs)}", + $"intermediate_values={PythonRepr.DictRepr(IntermediateValuesForRead)}", + $"distributions={PythonRepr.DictRepr((IDictionary)_distributions)}", $"trial_id={TrialId}", "value=None", }; diff --git a/src/OptunaSharp.Core/Importance/BaseImportanceEvaluator.cs b/src/OptunaSharp.Core/Importance/BaseImportanceEvaluator.cs index bb8c8e4..881e047 100644 --- a/src/OptunaSharp.Core/Importance/BaseImportanceEvaluator.cs +++ b/src/OptunaSharp.Core/Importance/BaseImportanceEvaluator.cs @@ -89,7 +89,7 @@ internal static OrderedDict GetDistributions( OrderedDict? distributions = null; foreach (var trial in completedTrials) { - var trialDistributions = trial.DistributionsDict; + var trialDistributions = trial.Distributions; if (!@params.All(name => trialDistributions.ContainsKey(name))) continue; @@ -131,7 +131,7 @@ internal static void CheckEvaluateArgs( if (@params is not null && @params.Count > 0) { bool atLeastOneTrial = completedTrials.Any( - trial => @params.All(p => trial.DistributionsDict.ContainsKey(p))); + trial => @params.All(p => trial.Distributions.ContainsKey(p))); if (!atLeastOneTrial) throw new ValueError( "Study must contain completed trials with all specified parameters. " + diff --git a/src/OptunaSharp.Core/Importance/PedAnova/PedAnovaImportanceEvaluator.cs b/src/OptunaSharp.Core/Importance/PedAnova/PedAnovaImportanceEvaluator.cs index 8f3517a..b38c02c 100644 --- a/src/OptunaSharp.Core/Importance/PedAnova/PedAnovaImportanceEvaluator.cs +++ b/src/OptunaSharp.Core/Importance/PedAnova/PedAnovaImportanceEvaluator.cs @@ -272,7 +272,7 @@ private static OrderedDict ZeroImportances(IReadOnlyList var regimeTrials = new List<(BaseDistribution? Dist, List Trials)>(); foreach (var trial in trials) { - BaseDistribution? key = trial.DistributionsDict.TryGetValue(paramName, out var d) ? d : null; + BaseDistribution? key = trial.Distributions.TryGetValue(paramName, out var d) ? d : null; int found = -1; for (int i = 0; i < regimeTrials.Count; i++) { @@ -325,7 +325,7 @@ private static List ResolveParams(Study study, IReadOnlyList? @p var allParams = new List(); foreach (var trial in trials) { - foreach (var name in trial.DistributionsDict.Keys) + foreach (var name in trial.Distributions.Keys) { if (seen.Add(name)) allParams.Add(name); diff --git a/src/OptunaSharp.Core/Importance/PedAnova/ScottParzenEstimator.cs b/src/OptunaSharp.Core/Importance/PedAnova/ScottParzenEstimator.cs index 1c7ffa6..4efdda4 100644 --- a/src/OptunaSharp.Core/Importance/PedAnova/ScottParzenEstimator.cs +++ b/src/OptunaSharp.Core/Importance/PedAnova/ScottParzenEstimator.cs @@ -274,7 +274,7 @@ internal static long[] CountNumericalParamInGrid( var paramValues = new double[trials.Count]; for (int t = 0; t < trials.Count; t++) { - PyFloat.TryConvert(trials[t].ParamsDict[paramName], out double v); + PyFloat.TryConvert(trials[t].Params[paramName], out double v); paramValues[t] = log ? Math.Log(v) : v; } @@ -320,7 +320,7 @@ internal static long[] CountCategoricalParamInGrid( var counts = new long[dist.Choices.Count]; foreach (var trial in trials) { - int index = (int)dist.ToInternalRepr(trial.ParamsDict[paramName]); + int index = (int)dist.ToInternalRepr(trial.Params[paramName]); counts[index] += 1; } return counts; diff --git a/src/OptunaSharp.Core/Internal/CompactTrialParts.cs b/src/OptunaSharp.Core/Internal/CompactTrialParts.cs new file mode 100644 index 0000000..cc9c22b --- /dev/null +++ b/src/OptunaSharp.Core/Internal/CompactTrialParts.cs @@ -0,0 +1,963 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using OptunaSharp.Collections; +using OptunaSharp.Distributions; + +namespace OptunaSharp.Internal; + +/// +/// A finished trial's dictionary held in a compact form (Rustuna plan 13, item S12): readable in place through +/// , and turned back into the +/// it was built from when a caller asks a for its mutable dictionary. +/// +/// The dictionary's key type. +/// The dictionary's value type. +/// +/// Only built with compactFinishedTrials: true produces these. Every +/// implementation is immutable after construction, so any number of threads may read one, and a trial may share one +/// with its or with every other trial of the same parameter schema. +/// +internal interface ICompactDictionary : IReadOnlyDictionary + where TKey : notnull +{ + /// A new dictionary holding the same entries in the same order, owned by the caller. + /// A fresh sized to the entry count. + /// + /// Values are rebuilt the way a read rebuilds them: a parameter's external value is a new box with the stored + /// bits, a compacted string a new string with the stored characters. Equal to the original entries, not the same + /// objects; nested containers in attributes are the same objects. + /// + OrderedDict Materialize(); +} + +/// +/// The read-only dictionary surface every compact form shares: lookups, ordered enumeration, and the non-generic +/// that and recognize as a Python dict. +/// +/// The dictionary's key type. +/// The dictionary's value type. +/// +/// +/// Mutation through throws , which is the point: +/// a caller that wants to change a compact trial must go through the trial's …Dict accessor, which +/// replaces the compact form with a real dictionary first, so the change is kept. A cast to +/// fails instead of writing into a copy that would be thrown away. +/// +/// +/// Entries enumerate in the order the source dictionary held them, which is what Optuna's insertion-ordered +/// dicts promise. +/// +/// +internal abstract class CompactDictionaryView : ICompactDictionary, IDictionary + where TKey : notnull +{ + /// Number of entries. + public abstract int Count { get; } + + /// The key at an enumeration position. + /// Position in [0, Count). + /// The key stored at that position. + protected abstract TKey KeyAt(int index); + + /// The value at an enumeration position, rebuilt from the compact form if it is stored compactly. + /// Position in [0, Count). + /// The value a reader of the original dictionary would have seen, possibly as a new object. + protected abstract TValue ValueAt(int index); + + /// The position of a key, or -1 when the dictionary does not hold it. + /// Key to find, compared ordinally like the source dictionary's default comparer. + /// The position in [0, Count), or -1. + protected abstract int IndexOf(TKey key); + + /// The value stored under . + /// Key to look up. + /// The value a reader of the original dictionary would get, rebuilt if it is stored compactly. + /// is null. + /// The dictionary does not hold ; the message is the BCL dictionaries' own. + public TValue this[TKey key] + { + get + { + ArgumentNullException.ThrowIfNull(key); + int index = IndexOf(key); + if (index < 0) + throw new KeyNotFoundException($"The given key '{key}' was not present in the dictionary."); + return ValueAt(index); + } + } + + /// The keys in enumeration order. + public virtual IEnumerable Keys + { + get + { + for (int i = 0; i < Count; i++) + yield return KeyAt(i); + } + } + + /// The values in enumeration order, each rebuilt as rebuilds it. + public IEnumerable Values + { + get + { + for (int i = 0; i < Count; i++) + yield return ValueAt(i); + } + } + + /// Whether the dictionary holds . + /// Key to look up. + /// True when the key is present. + /// is null. + public bool ContainsKey(TKey key) + { + ArgumentNullException.ThrowIfNull(key); + return IndexOf(key) >= 0; + } + + /// Looks up without throwing when it is absent. + /// Key to look up. + /// The value when present, otherwise the type's default. + /// True when the key is present. + /// is null. + public bool TryGetValue(TKey key, out TValue value) + { + ArgumentNullException.ThrowIfNull(key); + int index = IndexOf(key); + if (index < 0) + { + value = default!; + return false; + } + value = ValueAt(index); + return true; + } + + /// Enumerates the entries in order, rebuilding each value as it is reached. + /// An enumerator over key/value pairs. + public IEnumerator> GetEnumerator() + { + for (int i = 0; i < Count; i++) + yield return new KeyValuePair(KeyAt(i), ValueAt(i)); + } + + /// Enumerates the entries in order. + /// An enumerator over key/value pairs. + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// + public OrderedDict Materialize() + { + var dictionary = new OrderedDict(Count); + for (int i = 0; i < Count; i++) + dictionary.Add(KeyAt(i), ValueAt(i)); + return dictionary; + } + + // ------------------------------------------------------------------ IDictionary (read-only) + + /// Always true: no entry can be added or removed. + bool IDictionary.IsFixedSize => true; + + /// Always true: see the class remarks for how to change a compact trial. + bool IDictionary.IsReadOnly => true; + + /// A snapshot array of the keys in enumeration order. + ICollection IDictionary.Keys + { + get + { + var keys = new TKey[Count]; + for (int i = 0; i < keys.Length; i++) + keys[i] = KeyAt(i); + return keys; + } + } + + /// A snapshot array of the values in enumeration order. + ICollection IDictionary.Values + { + get + { + var values = new TValue[Count]; + for (int i = 0; i < values.Length; i++) + values[i] = ValueAt(i); + return values; + } + } + + /// The value under a key, or null when absent or of another key type (the non-generic contract). + /// Key to look up. + /// The value, rebuilt if stored compactly, or null. + /// is null. + /// On set: the dictionary is read-only. + object? IDictionary.this[object key] + { + get + { + ArgumentNullException.ThrowIfNull(key); + if (key is TKey typed) + { + int index = IndexOf(typed); + if (index >= 0) + return ValueAt(index); + } + return null; + } + set => throw ReadOnly(); + } + + /// Not supported: the dictionary is read-only. + /// Ignored. + /// Ignored. + /// Always. + void IDictionary.Add(object key, object? value) => throw ReadOnly(); + + /// Not supported: the dictionary is read-only. + /// Always. + void IDictionary.Clear() => throw ReadOnly(); + + /// Whether a key of the dictionary's key type is present. + /// Key to look up. + /// True when present. + /// is null. + bool IDictionary.Contains(object key) + { + ArgumentNullException.ThrowIfNull(key); + return key is TKey typed && IndexOf(typed) >= 0; + } + + /// A non-generic enumerator yielding values in order. + /// The enumerator. + IDictionaryEnumerator IDictionary.GetEnumerator() => new EntryEnumerator(this); + + /// Not supported: the dictionary is read-only. + /// Ignored. + /// Always. + void IDictionary.Remove(object key) => throw ReadOnly(); + + /// Copies the entries, as key/value pairs or values, into an array. + /// A one-dimensional array of , or . + /// First position written. + /// is null. + /// is negative. + /// The array is too short or of an element type that cannot hold an entry. + void ICollection.CopyTo(Array array, int index) + { + ArgumentNullException.ThrowIfNull(array); + ArgumentOutOfRangeException.ThrowIfNegative(index); + if (array.Length - index < Count) + throw new ArgumentException("Destination array is not long enough to copy all the items in the collection."); + switch (array) + { + case KeyValuePair[] pairs: + for (int i = 0; i < Count; i++) + pairs[index + i] = new KeyValuePair(KeyAt(i), ValueAt(i)); + return; + case DictionaryEntry[] entries: + for (int i = 0; i < Count; i++) + entries[index + i] = new DictionaryEntry(KeyAt(i), ValueAt(i)); + return; + case object?[] objects: + for (int i = 0; i < Count; i++) + objects[index + i] = new KeyValuePair(KeyAt(i), ValueAt(i)); + return; + default: + throw new ArgumentException("Target array type is not compatible with the type of items in the collection."); + } + } + + /// The entry count. + int ICollection.Count => Count; + + /// False: reads need no synchronization because the view never changes. + bool ICollection.IsSynchronized => false; + + /// This view itself. + object ICollection.SyncRoot => this; + + /// The exception every mutation through throws. + /// A naming the way to change a compact trial. + private static NotSupportedException ReadOnly() + => new("This dictionary is a finished trial's compact form and is read-only; change the trial through its " + + "ParamsDict / DistributionsDict / UserAttrsDict / SystemAttrsDict accessor instead."); + + /// The non-generic enumerator over a view, in entry order. + private sealed class EntryEnumerator : IDictionaryEnumerator + { + private readonly CompactDictionaryView _view; + private int _index = -1; + private DictionaryEntry _current; + + /// Starts before the first entry of . + /// The view to enumerate. + internal EntryEnumerator(CompactDictionaryView view) => _view = view; + + /// The current entry. + public DictionaryEntry Entry => _current; + + /// The current key. + public object Key => _current.Key; + + /// The current value. + public object? Value => _current.Value; + + /// The current entry, boxed. + public object Current => _current; + + /// Advances to the next entry. + /// False past the last entry. + public bool MoveNext() + { + if (_index + 1 >= _view.Count) + { + _index = _view.Count; + return false; + } + _index++; + _current = new DictionaryEntry(_view.KeyAt(_index), _view.ValueAt(_index)); + return true; + } + + /// Returns to the position before the first entry. + public void Reset() + { + _index = -1; + _current = default; + } + } +} + +/// +/// The ordered parameter names and distribution instances that many finished trials of a study share (plan 13, item +/// S12): a compact trial stores only its internal values against one of these. +/// +/// +/// Immutable, so trials and threads share it freely. Two trials share a schema only when their parameter names come +/// in the same order and their distributions are the same instances; already +/// shares bitwise-identical distributions within a study (plan 13, item S7), so a static search space yields one +/// schema for the whole study. +/// +internal sealed class ParamSchema +{ + private readonly Dictionary _index; + + /// Builds a schema from parallel arrays, which it keeps (the caller hands over ownership). + /// Parameter names in the trials' order; never changed after this call. + /// The distribution instance of each name, in the same order. + internal ParamSchema(string[] names, BaseDistribution[] distributions) + { + Names = names; + Distributions = distributions; + _index = new Dictionary(names.Length); + for (int i = 0; i < names.Length; i++) + _index[names[i]] = i; + ReadOnlyNames = new ReadOnlyCollection(names); + View = new DistributionsView(this); + } + + /// The parameter names in order. Not to be changed: every trial of the schema reads it. + internal string[] Names { get; } + + /// The distribution of each name, in order. Not to be changed, as . + internal BaseDistribution[] Distributions { get; } + + /// The names wrapped read-only, handed out as a view's Keys so no caller can change the array. + internal ReadOnlyCollection ReadOnlyNames { get; } + + /// The one read-only distributions dictionary every trial of this schema returns from Distributions. + internal DistributionsView View { get; } + + /// The position of a parameter name, or -1. + /// Parameter name, compared ordinally like the trials' dictionaries. + /// Its position, or -1 when the schema does not hold it. + internal int IndexOf(string name) => _index.TryGetValue(name, out int index) ? index : -1; + + /// Marks, in , a parameter the schema does not hold. + internal const int Absent = -1; + + /// + /// Marks, in , a parameter the schema holds under a distribution that may convert + /// values differently from the reader's, so only rebuilding the external value is exact. + /// + internal const int NotInterchangeable = -2; + + /// + /// For each parameter a reader converts with its own distribution, where a compact trial of this schema stores a + /// value that is exactly what that conversion would return (plan 13, item S12). + /// + /// The parameters the reader wants, in its order. + /// The distribution the reader converts each one with, in the same order. + /// + /// Per parameter: its position in the schema when the reader's distribution is interchangeable with the schema's + /// (), so equals + /// distribution.ToInternalRepr(Params[name]); when the schema lacks it; + /// otherwise. + /// + /// + /// Why the stored value is exact: compaction keeps internal = d.ToInternalRepr(stored) only when + /// d.ToExternalRepr(internal) is indistinguishable from stored, and an interchangeable distribution + /// converts every value as d does, so it maps the rebuilt external value back to that same internal. + /// + internal int[] PositionsFor(string[] names, BaseDistribution[] distributions) + { + var positions = new int[names.Length]; + for (int i = 0; i < names.Length; i++) + { + int position = IndexOf(names[i]); + positions[i] = position < 0 + ? Absent + : DistributionIdentity.AreInterchangeable(distributions[i], Distributions[position]) + ? position + : NotInterchangeable; + } + return positions; + } + + /// + /// Whether a trial's dictionaries have this schema: the same names in the same order in both, and distributions + /// interchangeable with the schema's (). + /// + /// The trial's parameter dictionary. + /// The trial's distribution dictionary. + /// True when the trial can store its values against this schema. + internal bool Matches(OrderedDict @params, OrderedDict distributions) + { + if (@params.Count != Names.Length || distributions.Count != Names.Length) + return false; + for (int i = 0; i < Names.Length; i++) + { + KeyValuePair entry = distributions.GetAt(i); + if (!string.Equals(entry.Key, Names[i], StringComparison.Ordinal) + || !string.Equals(@params.GetAt(i).Key, Names[i], StringComparison.Ordinal) + || !DistributionIdentity.AreInterchangeable(entry.Value, Distributions[i])) + return false; + } + return true; + } + + /// + /// The hash a files a schema under: names by value and distributions by + /// , in order. + /// + /// A trial's distribution dictionary. + /// A hash consistent with : trials that match one schema hash alike. + internal static int HashOf(OrderedDict distributions) + { + var hash = new HashCode(); + hash.Add(distributions.Count); + for (int i = 0; i < distributions.Count; i++) + { + KeyValuePair entry = distributions.GetAt(i); + hash.Add(entry.Key, StringComparer.Ordinal); + hash.Add(DistributionIdentity.HashOf(entry.Value)); + } + return hash.ToHashCode(); + } +} + +/// +/// When two distribution objects can stand for each other in a compact trial (plan 13, item S12): no read can tell +/// them apart except by asking whether they are the same object. +/// +/// +/// +/// Float and integer distributions follow , the rule the storage +/// already shares instances by (plan 13, item S7). A plain qualifies when +/// its choices are identical one by one (): every SuggestCategorical call builds +/// a new distribution, so without this a categorical parameter would give every trial a schema of its own. Python-equal choices that +/// print differently (1 and 1.0) are not identical. +/// +/// +/// Anything else (a subclass of the categorical distribution, a custom distribution) must be the same instance. +/// +/// +internal static class DistributionIdentity +{ + /// Whether can be returned where a trial stored , and the other way round. + /// One distribution. + /// The other. + /// True for the same instance or indistinguishable ones (see the class remarks). + internal static bool AreInterchangeable(BaseDistribution a, BaseDistribution b) + { + if (ReferenceEquals(a, b) || Distribution.IsBitwiseIdentical(a, b)) + return true; + if (a.GetType() != typeof(CategoricalDistribution) || b.GetType() != typeof(CategoricalDistribution)) + return false; + IReadOnlyList choicesA = ((CategoricalDistribution)a).Choices; + IReadOnlyList choicesB = ((CategoricalDistribution)b).Choices; + if (choicesA.Count != choicesB.Count) + return false; + for (int i = 0; i < choicesA.Count; i++) + if (!IsSameValue(choicesA[i].Value, choicesB[i].Value)) + return false; + return true; + } + + /// A hash consistent with : interchangeable distributions hash alike. + /// The distribution to hash. + /// Its field-wise hash for the kinds compared by content, its identity hash otherwise. + internal static int HashOf(BaseDistribution distribution) + { + switch (distribution) + { + case FloatDistribution f: + return HashCode.Combine(f.GetType(), BitConverter.DoubleToInt64Bits(f.Low), + BitConverter.DoubleToInt64Bits(f.High), f.Step is null ? 0L : BitConverter.DoubleToInt64Bits(f.Step.Value), + f.Log); + case IntDistribution i: + return HashCode.Combine(i.GetType(), i.Low, i.High, i.Step, i.Log); + case CategoricalDistribution c when c.GetType() == typeof(CategoricalDistribution): + var hash = new HashCode(); + hash.Add(c.Choices.Count); + foreach (CategoricalChoice choice in c.Choices) + hash.Add(ValueHash(choice.Value)); + return hash.ToHashCode(); + default: + return RuntimeHelpers.GetHashCode(distribution); + } + } + + /// + /// Whether two values are indistinguishable except by object identity: the same object, or values of the same + /// type with the same bits (floating point), the same number (integers), the same truth value, or the same + /// characters (strings). + /// + /// One value. + /// The other. + /// True when a reader comparing anything but references sees one value. + /// + /// Bits, not ==, for floating point: -0.0 and 0.0 are equal but print differently. Any other kind of + /// object (a decimal, whose 1.0m and 1.00m are equal but print differently, a list, a user type) + /// must be the same instance. + /// + internal static bool IsSameValue(object? a, object? b) + { + if (ReferenceEquals(a, b)) + return true; + if (a is null || b is null || a.GetType() != b.GetType()) + return false; + return a switch + { + double x => BitConverter.DoubleToInt64Bits(x) == BitConverter.DoubleToInt64Bits((double)b), + float x => BitConverter.SingleToInt32Bits(x) == BitConverter.SingleToInt32Bits((float)b), + string x => string.Equals(x, (string)b, StringComparison.Ordinal), + long or int or short or sbyte or byte or ulong or uint or ushort or bool or char => a.Equals(b), + _ => false, + }; + } + + /// A hash consistent with . + /// The value to hash. + /// A content hash for the kinds compared by content, an identity hash otherwise. + private static int ValueHash(object? value) => value switch + { + null => 0, + double x => HashCode.Combine(typeof(double), BitConverter.DoubleToInt64Bits(x)), + float x => HashCode.Combine(typeof(float), BitConverter.SingleToInt32Bits(x)), + string x => HashCode.Combine(typeof(string), StringComparer.Ordinal.GetHashCode(x)), + long or int or short or sbyte or byte or ulong or uint or ushort or bool or char + => HashCode.Combine(value.GetType(), value.GetHashCode()), + _ => RuntimeHelpers.GetHashCode(value), + }; +} + +/// +/// A study's parameter schemas (plan 13, item S12), found or created as its trials finish. Not thread-safe: +/// uses it under its lock. +/// +/// +/// A static search space hits 's last-schema check every time, which costs one pass over the +/// names. A dynamic one keeps one schema per distinct shape up to ; trials of further +/// shapes keep their dictionaries, which only forgoes the memory saving. +/// +internal sealed class ParamSchemaCache +{ + /// The most schemas one study keeps; enough for conditional spaces, bounded for pathological ones. + internal const int MaxSchemas = 256; + + private readonly Dictionary> _byHash = new(); + private ParamSchema? _last; + + /// How many schemas the study has; a diagnostic for tests. + internal int Count { get; private set; } + + /// + /// The schema a finished trial's dictionaries have, created on first sight, or null when the trial cannot use + /// one. + /// + /// The trial's parameter dictionary. + /// The trial's distribution dictionary. + /// + /// The matching schema, or null when the two dictionaries do not hold the same names in the same order (a trial + /// added from a template may list them differently) or the study already has . + /// + internal ParamSchema? FindOrAdd(OrderedDict @params, OrderedDict distributions) + { + if (_last is not null && _last.Matches(@params, distributions)) + return _last; + if (@params.Count != distributions.Count) + return null; + + int hash = ParamSchema.HashOf(distributions); + if (_byHash.TryGetValue(hash, out List? candidates)) + { + foreach (ParamSchema candidate in candidates) + { + if (candidate.Matches(@params, distributions)) + return _last = candidate; + } + } + + // A new shape: the parameter names must come in the distributions' order, or one schema cannot rebuild both. + int n = distributions.Count; + var names = new string[n]; + var instances = new BaseDistribution[n]; + for (int i = 0; i < n; i++) + { + KeyValuePair entry = distributions.GetAt(i); + if (!string.Equals(@params.GetAt(i).Key, entry.Key, StringComparison.Ordinal)) + return null; + names[i] = entry.Key; + instances[i] = entry.Value; + } + if (Count >= MaxSchemas) + return null; + + var schema = new ParamSchema(names, instances); + if (candidates is null) + _byHash[hash] = candidates = new List(1); + candidates.Add(schema); + Count++; + return _last = schema; + } +} + +/// +/// A compact trial's parameters (plan 13, item S12): one internal-representation value per name of a shared +/// , read back through each distribution's ToExternalRepr. +/// +/// +/// accepts a trial only when every stored value comes back indistinguishable from its +/// internal form (: same type and bits for numbers, same characters for +/// strings, the same object otherwise), so a read returns what the trial stored; anything else keeps its dictionary. +/// A read of a numeric value allocates a new box each time. +/// +internal sealed class CompactParams : CompactDictionaryView +{ + private readonly double[] _internals; + + /// Wraps values already verified by . + /// The shared names and distributions. + /// One internal value per schema entry; owned by this object from now on. + private CompactParams(ParamSchema schema, double[] internals) + { + Schema = schema; + _internals = internals; + } + + /// The shared names and distributions the values are stored against. + internal ParamSchema Schema { get; } + + /// + public override int Count => _internals.Length; + + /// The schema's names, read-only. + public override IEnumerable Keys => Schema.ReadOnlyNames; + + /// + protected override string KeyAt(int index) => Schema.Names[index]; + + /// + protected override object? ValueAt(int index) => Schema.Distributions[index].ToExternalRepr(_internals[index]); + + /// + protected override int IndexOf(string key) => Schema.IndexOf(key); + + /// + /// The stored internal value at a schema position, without rebuilding the external value: what + /// d.ToInternalRepr(Params[name]) returns for the schema's distribution d or any distribution + /// interchangeable with it (see ). + /// + /// A position returned for this trial's schema. + /// The internal value stored at that position. + /// is not a position of the schema. + internal double InternalAt(int position) => _internals[position]; + + /// + /// The stored internal value of a parameter without rebuilding its external value: what + /// Distributions[name].ToInternalRepr(Params[name]) returns, since stored exactly + /// that. + /// + /// Parameter name. + /// The internal value, or 0 when absent. + /// True when the trial has the parameter. + internal bool TryGetInternal(string name, out double value) + { + int index = Schema.IndexOf(name); + if (index < 0) + { + value = 0; + return false; + } + value = _internals[index]; + return true; + } + + /// + /// The compact form of a finished trial's parameters, or null when any value would not read back identical. + /// + /// The schema matched to the trial. + /// The trial's parameter dictionary, in the schema's order. + /// The compact parameters, or null to keep the dictionary. + /// + /// A value a trial got from SetTrialParam is the distribution's own ToExternalRepr of its internal + /// value and always qualifies. A value from a template trial may not: an int under a float distribution + /// (Optuna keeps it an int), a 64-bit integer beyond 2^53, or a categorical choice equal to an earlier one + /// (1 and 1.0), whose first index is the one ToInternalRepr finds. + /// + internal static CompactParams? TryCreate(ParamSchema schema, OrderedDict @params) + { + var internals = new double[schema.Names.Length]; + for (int i = 0; i < internals.Length; i++) + { + object? stored = @params.GetAt(i).Value; + BaseDistribution distribution = schema.Distributions[i]; + double internalRepr; + try + { + internalRepr = distribution.ToInternalRepr(stored); + } + catch (Exception) + { + // A value its own distribution cannot convert (only a hand-built template can hold one) keeps the + // dictionary as it is; compaction must never fail the finish that triggered it. + return null; + } + // The rebuilt value must be indistinguishable from the stored one; the schema's distribution may be an + // interchangeable copy of the trial's own, whose choice objects then differ only by identity. + if (!DistributionIdentity.IsSameValue(stored, distribution.ToExternalRepr(internalRepr))) + return null; + internals[i] = internalRepr; + } + return new CompactParams(schema, internals); + } +} + +/// +/// The read-only distributions dictionary of every compact trial of one (plan 13, item +/// S12): one instance per schema, not per trial. +/// +internal sealed class DistributionsView : CompactDictionaryView +{ + private readonly ParamSchema _schema; + + /// Wraps a schema; created once by the schema itself. + /// The schema whose names and distributions this view exposes. + internal DistributionsView(ParamSchema schema) => _schema = schema; + + /// + public override int Count => _schema.Names.Length; + + /// The schema's names, read-only. + public override IEnumerable Keys => _schema.ReadOnlyNames; + + /// + protected override string KeyAt(int index) => _schema.Names[index]; + + /// + protected override BaseDistribution ValueAt(int index) => _schema.Distributions[index]; + + /// + protected override int IndexOf(string key) => _schema.IndexOf(key); +} + +/// +/// A compact trial's user or system attributes (plan 13, item S12): two arrays instead of a dictionary, with long +/// ASCII strings held as bytes (), and one shared instance for every empty one. +/// +/// +/// Values other than long strings are kept as they are, so a nested list or dictionary is the same object the trial +/// stored. Lookups scan the keys while there are few of them and use an index built on the first lookup otherwise. +/// +internal sealed class CompactAttrs : CompactDictionaryView +{ + /// Up to this many keys a lookup scans; more build an index once. + internal const int LinearScanLimit = 16; + + /// The one compact form of every empty attribute dictionary. + internal static readonly CompactAttrs Empty = new(Array.Empty(), Array.Empty()); + + private readonly string[] _keys; + private readonly object?[] _values; + private Dictionary? _index; + + /// Wraps the arrays built. + /// Keys in order; owned by this object. + /// Values in order, long strings as ; owned by this object. + private CompactAttrs(string[] keys, object?[] values) + { + _keys = keys; + _values = values; + } + + /// + public override int Count => _keys.Length; + + /// + protected override string KeyAt(int index) => _keys[index]; + + /// + protected override object? ValueAt(int index) + => _values[index] is CompactString compact ? compact.Materialize() : _values[index]; + + /// The value as stored, a for a compacted string; a diagnostic for tests. + /// An attribute name. + /// The stored object, or null when the key is absent. + internal object? StoredValue(string key) + { + int index = IndexOf(key); + return index < 0 ? null : _values[index]; + } + + /// + protected override int IndexOf(string key) + { + if (_keys.Length <= LinearScanLimit) + { + for (int i = 0; i < _keys.Length; i++) + if (string.Equals(_keys[i], key, StringComparison.Ordinal)) + return i; + return -1; + } + + // Built once per object; two readers may both build one, and either result is correct, so the race only + // costs a duplicate build. Volatile so a reader never sees a dictionary before its entries. + Dictionary? index = Volatile.Read(ref _index); + if (index is null) + { + index = new Dictionary(_keys.Length); + for (int i = 0; i < _keys.Length; i++) + index[_keys[i]] = i; + Volatile.Write(ref _index, index); + } + return index.TryGetValue(key, out int position) ? position : -1; + } + + /// The compact form of an attribute dictionary. + /// The dictionary; read, never changed. + /// for an empty dictionary, otherwise a new compact copy of its entries. + internal static CompactAttrs Create(OrderedDict attrs) + { + int n = attrs.Count; + if (n == 0) + return Empty; + var keys = new string[n]; + var values = new object?[n]; + for (int i = 0; i < n; i++) + { + KeyValuePair entry = attrs.GetAt(i); + keys[i] = entry.Key; + values[i] = entry.Value is string text ? CompactString.TryCreate(text) ?? (object)text : entry.Value; + } + return new CompactAttrs(keys, values); + } +} + +/// +/// A long ASCII string held as bytes (plan 13, item S12): half the memory of the UTF-16 string, or a quarter for a +/// lowercase hexadecimal string such as the CMA-ES optimizer state Optuna stores in system attributes. +/// +/// +/// +/// returns a new string with exactly the original characters on every call; a form is +/// chosen only when that round trip is exact. Hexadecimal digits are stored two to a byte, and an odd count keeps +/// its last digit aside: CMA-ES splits its state into 2,045-character chunks, an odd length. +/// +/// +internal sealed class CompactString +{ + /// Strings shorter than this stay strings: the saving would not cover the extra object. + internal const int MinLength = 64; + + private readonly byte[] _bytes; + private readonly bool _lowerHex; + private readonly char _trailingDigit; + + /// Wraps the stored bytes. + /// ASCII characters, or the decoded digit pairs of a lowercase hexadecimal string. + /// True when must be re-encoded as lowercase hexadecimal. + /// The last digit of an odd-length hexadecimal string, or '\0' for none. + private CompactString(byte[] bytes, bool lowerHex, char trailingDigit) + { + _bytes = bytes; + _lowerHex = lowerHex; + _trailingDigit = trailingDigit; + } + + /// Whether this string is stored as hexadecimal digit pairs; a diagnostic for tests. + internal bool IsHex => _lowerHex; + + /// How many bytes the stored form holds; a diagnostic for tests. + internal int StoredBytes => _bytes.Length; + + /// The compact form of a string, or null when it is short or not ASCII. + /// The string to store. + /// A compact string whose returns 's characters, or null. + internal static CompactString? TryCreate(string text) + { + if (text.Length < MinLength) + return null; + if (IsLowerHex(text)) + { + int pairs = text.Length / 2; + return new CompactString( + Convert.FromHexString(text.AsSpan(0, pairs * 2)), + lowerHex: true, + trailingDigit: (text.Length & 1) != 0 ? text[^1] : '\0'); + } + if (!Ascii.IsValid(text)) + return null; + return new CompactString(Encoding.ASCII.GetBytes(text), lowerHex: false, trailingDigit: '\0'); + } + + /// A new string with the stored characters. + /// The original string's characters, as a new instance. + internal string Materialize() + { + if (!_lowerHex) + return Encoding.ASCII.GetString(_bytes); + int length = _bytes.Length * 2 + (_trailingDigit != '\0' ? 1 : 0); + return string.Create(length, this, static (chars, compact) => + { + const string digits = "0123456789abcdef"; + byte[] bytes = compact._bytes; + for (int i = 0; i < bytes.Length; i++) + { + chars[2 * i] = digits[bytes[i] >> 4]; + chars[2 * i + 1] = digits[bytes[i] & 0xF]; + } + if (compact._trailingDigit != '\0') + chars[^1] = compact._trailingDigit; + }); + } + + /// Whether every character is a lowercase hexadecimal digit, so bytes and one spare digit rebuild it exactly. + /// The string to test. + /// True for [0-9a-f] only; uppercase digits would come back lowercase, so they do not qualify. + private static bool IsLowerHex(string text) + { + foreach (char c in text) + { + if (c is not ((>= '0' and <= '9') or (>= 'a' and <= 'f'))) + return false; + } + return true; + } +} diff --git a/src/OptunaSharp.Core/Internal/ConstrainedOptimization.cs b/src/OptunaSharp.Core/Internal/ConstrainedOptimization.cs index c39f32a..5251392 100644 --- a/src/OptunaSharp.Core/Internal/ConstrainedOptimization.cs +++ b/src/OptunaSharp.Core/Internal/ConstrainedOptimization.cs @@ -51,15 +51,18 @@ internal static OrderedDict GetConstraintsFromSystemAttrs( } // Load constraints from new format (individual keys). + // Keys first, the value only for a match: the same entries in the same order as walking the pairs, but a + // compact finished trial (plan 13, item S12) then rebuilds no stored string it does not need. TPE reads every + // trial's constraints on every ask, and its trials carry their relative parameters as long JSON strings. string prefix = ConstraintsKey + ":"; - foreach (var (key, value) in systemAttrs) + foreach (string key in systemAttrs.Keys) { if (key.StartsWith(prefix, System.StringComparison.Ordinal)) { string constraintKey = key.Substring(prefix.Length); if (constraintsDict.ContainsKey(constraintKey)) OptunaWarnings.Warn("Overwrite an old format constraint."); - constraintsDict[constraintKey] = ToDouble(value); + constraintsDict[constraintKey] = ToDouble(systemAttrs[key]); } } return constraintsDict; diff --git a/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs b/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs index d681683..41e34be 100644 --- a/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs +++ b/src/OptunaSharp.Core/Internal/TpeTrialSplit.cs @@ -162,10 +162,10 @@ private static (List Below, List Above) SplitCompleteT /// A lexicographic score prioritizing later pruning steps and then better intermediate values. private static (double, double) GetPrunedTrialScore(FrozenTrial trial, Study study) { - if (trial.IntermediateValues.Count > 0) + if (trial.IntermediateValuesForRead.Count > 0) { - long step = trial.IntermediateValues.Keys.Max(); // max(dict.items()) compares by key (step). - double intermediateValue = trial.IntermediateValues[step]; + long step = trial.IntermediateValuesForRead.Keys.Max(); // max(dict.items()) compares by key (step). + double intermediateValue = trial.IntermediateValuesForRead[step]; if (double.IsNaN(intermediateValue)) return (-step, double.PositiveInfinity); if (study.Directions[0] == StudyDirection.Minimize) diff --git a/src/OptunaSharp.Core/Internal/TrialsDataframe.cs b/src/OptunaSharp.Core/Internal/TrialsDataframe.cs index b4d93e0..57f5826 100644 --- a/src/OptunaSharp.Core/Internal/TrialsDataframe.cs +++ b/src/OptunaSharp.Core/Internal/TrialsDataframe.cs @@ -164,7 +164,7 @@ private static (List> Records, List<(strin } case "params": { - foreach (var (nestedAttr, nestedValue) in trial.ParamsDict) + foreach (var (nestedAttr, nestedValue) in trial.Params) { record[(dfColumn, nestedAttr)] = nestedValue; Agg().Add((dfColumn, nestedAttr)); @@ -173,7 +173,7 @@ private static (List> Records, List<(strin } case "user_attrs": { - foreach (var (nestedAttr, nestedValue) in trial.UserAttrsDict) + foreach (var (nestedAttr, nestedValue) in trial.UserAttrs) { record[(dfColumn, nestedAttr)] = nestedValue; Agg().Add((dfColumn, nestedAttr)); @@ -182,7 +182,7 @@ private static (List> Records, List<(strin } case "system_attrs": { - foreach (var (nestedAttr, nestedValue) in trial.SystemAttrsDict) + foreach (var (nestedAttr, nestedValue) in trial.SystemAttrs) { record[(dfColumn, nestedAttr)] = nestedValue; Agg().Add((dfColumn, nestedAttr)); @@ -191,7 +191,7 @@ private static (List> Records, List<(strin } case "intermediate_values": { - foreach (var (step, nestedValue) in trial.IntermediateValues) + foreach (var (step, nestedValue) in trial.IntermediateValuesForRead) { string nestedAttr = step.ToString(System.Globalization.CultureInfo.InvariantCulture); record[(dfColumn, nestedAttr)] = nestedValue; diff --git a/src/OptunaSharp.Core/Pruners/PercentilePruner.cs b/src/OptunaSharp.Core/Pruners/PercentilePruner.cs index 88bcc3b..537048a 100644 --- a/src/OptunaSharp.Core/Pruners/PercentilePruner.cs +++ b/src/OptunaSharp.Core/Pruners/PercentilePruner.cs @@ -157,7 +157,7 @@ internal static double GetPercentileIntermediateResultOverTrials( var intermediateValues = new List(); foreach (var t in completedTrials) { - if (t.IntermediateValues.TryGetValue(step, out double v)) + if (t.IntermediateValuesForRead.TryGetValue(step, out double v)) intermediateValues.Add(v); } diff --git a/src/OptunaSharp.Core/Pruners/SuccessiveHalvingPruner.cs b/src/OptunaSharp.Core/Pruners/SuccessiveHalvingPruner.cs index 7a4d66e..6cf17df 100644 --- a/src/OptunaSharp.Core/Pruners/SuccessiveHalvingPruner.cs +++ b/src/OptunaSharp.Core/Pruners/SuccessiveHalvingPruner.cs @@ -157,7 +157,7 @@ public override bool Prune(Study study, FrozenTrial trial) internal static int GetCurrentRung(FrozenTrial trial) { int rung = 0; - while (trial.SystemAttrsDict.ContainsKey(CompletedRungKey(rung))) + while (trial.SystemAttrs.ContainsKey(CompletedRungKey(rung))) rung += 1; return rung; } @@ -177,8 +177,8 @@ internal static List GetCompetingValues( IReadOnlyList trials, double value, string rungKey) { var competingValues = trials - .Where(t => t.SystemAttrsDict.ContainsKey(rungKey)) - .Select(t => Convert.ToDouble(t.SystemAttrsDict[rungKey], CultureInfo.InvariantCulture)) + .Where(t => t.SystemAttrs.ContainsKey(rungKey)) + .Select(t => Convert.ToDouble(t.SystemAttrs[rungKey], CultureInfo.InvariantCulture)) .ToList(); competingValues.Add(value); return competingValues; diff --git a/src/OptunaSharp.Core/Pruners/WilcoxonPruner.cs b/src/OptunaSharp.Core/Pruners/WilcoxonPruner.cs index dfda67f..0c46c62 100644 --- a/src/OptunaSharp.Core/Pruners/WilcoxonPruner.cs +++ b/src/OptunaSharp.Core/Pruners/WilcoxonPruner.cs @@ -86,7 +86,7 @@ public override bool Prune(Study study, FrozenTrial trial) return false; } - if (bestTrial.IntermediateValues.Count == 0) + if (bestTrial.IntermediateValuesForRead.Count == 0) { OptunaWarnings.Warn( "The best trial has no intermediate values so WilcoxonPruner cannot prune trials. " + @@ -95,8 +95,8 @@ public override bool Prune(Study study, FrozenTrial trial) return false; } - var bestSteps = bestTrial.IntermediateValues.Keys.ToArray(); - var bestStepValues = bestSteps.Select(s => bestTrial.IntermediateValues[s]).ToArray(); + var bestSteps = bestTrial.IntermediateValuesForRead.Keys.ToArray(); + var bestStepValues = bestSteps.Select(s => bestTrial.IntermediateValuesForRead[s]).ToArray(); if (bestStepValues.Any(v => !double.IsFinite(v))) { diff --git a/src/OptunaSharp.Core/Samplers/BaseGASampler.cs b/src/OptunaSharp.Core/Samplers/BaseGASampler.cs index 7a302f7..65bbd54 100644 --- a/src/OptunaSharp.Core/Samplers/BaseGASampler.cs +++ b/src/OptunaSharp.Core/Samplers/BaseGASampler.cs @@ -191,7 +191,7 @@ private void Fold(GenerationTracker tracker, FrozenTrial trial) if (trial.State != TrialState.Complete) return; // failed/pruned trials never belonged to a population in the reference scan either tracker.NumberById[trial.TrialId] = trial.Number; - object? raw = trial.SystemAttrsDict.GetValueOrDefault(GenerationKey); + object? raw = trial.SystemAttrs.GetValueOrDefault(GenerationKey); int generation = raw is null ? -1 : Convert.ToInt32(raw, CultureInfo.InvariantCulture); if (generation < 0) return; // the scan's `raw is null ? -1` case: contributes to no maximum and no population @@ -219,7 +219,7 @@ private void Fold(GenerationTracker tracker, FrozenTrial trial) /// The configured storage provider fails; its exception propagates. public int GetTrialGeneration(Study study, FrozenTrial trial) { - var stored = trial.SystemAttrsDict.GetValueOrDefault(GenerationKey); + var stored = trial.SystemAttrs.GetValueOrDefault(GenerationKey); if (stored is not null) return Convert.ToInt32(stored, CultureInfo.InvariantCulture); diff --git a/src/OptunaSharp.Core/Samplers/BruteForceSampler.cs b/src/OptunaSharp.Core/Samplers/BruteForceSampler.cs index e01f141..f8835b8 100644 --- a/src/OptunaSharp.Core/Samplers/BruteForceSampler.cs +++ b/src/OptunaSharp.Core/Samplers/BruteForceSampler.cs @@ -170,11 +170,11 @@ private static void WithPendingChoices( OrderedDict? distributions = null; foreach (var (name, (value, distribution)) in chosen) { - if (t.ParamsDict.ContainsKey(name)) + if (t.Params.ContainsKey(name)) continue; // Copied on the first missing value only: the storage's record is shared and read-only. - @params ??= new OrderedDict(t.ParamsDict); - distributions ??= new OrderedDict(t.DistributionsDict); + @params ??= new OrderedDict(t.Params); + distributions ??= new OrderedDict(t.Distributions); @params[name] = value; distributions[name] = distribution; } diff --git a/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs b/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs index 69dfae0..1f91871 100644 --- a/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs +++ b/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs @@ -337,7 +337,7 @@ public override OrderedDict InferRelativeSearchSpace(S double[] x = optimizer is CmaWm ? ((System.Collections.IEnumerable)t.SystemAttrs["x_for_tell"]!) .Cast().Select(v => Convert.ToDouble(v)).ToArray() - : trans.Transform(t.ParamsDict).ToArray(); + : trans.Transform(t.Params).ToArray(); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } @@ -585,7 +585,7 @@ private ICmaOptimizer InitOptimizer( { if (!expectedStates.Contains(t.State)) continue; if (!IsCompatibleSearchSpace(searchSpace, t.Distributions)) continue; - double[] x = trans.Transform(t.ParamsDict).ToArray(); + double[] x = trans.Transform(t.Params).ToArray(); sourceSolutions.Add((x, sign * t.Value!.Value)); } if (sourceSolutions.Count == 0) @@ -735,11 +735,11 @@ private void LogIndependentSampling(FrozenTrial trial, string paramName) { if (t.State == TrialState.Complete) return t; - if (t.State == TrialState.Pruned && t.IntermediateValues.Count > 0 && _considerPrunedTrials) + if (t.State == TrialState.Pruned && t.IntermediateValuesForRead.Count > 0 && _considerPrunedTrials) { // max(t.intermediate_values.items()) — the value at the largest step key. - long maxStep = t.IntermediateValues.Keys.Max(); - double value = t.IntermediateValues[maxStep]; + long maxStep = t.IntermediateValuesForRead.Keys.Max(); + double value = t.IntermediateValuesForRead[maxStep]; // We rewrite the value of the trial `t` for sampling, so we need a copy. var copied = t.DeepCopy(); copied.Value = value; diff --git a/src/OptunaSharp.Core/Samplers/DifferentialEvolutionSampler.cs b/src/OptunaSharp.Core/Samplers/DifferentialEvolutionSampler.cs index 5ced00b..d88b468 100644 --- a/src/OptunaSharp.Core/Samplers/DifferentialEvolutionSampler.cs +++ b/src/OptunaSharp.Core/Samplers/DifferentialEvolutionSampler.cs @@ -205,7 +205,7 @@ public override OrderedDict InferRelativeSearchSpace(S var solutions = new List<(double[] X, double Value)>(optimizer.PopulationSize); foreach (var t in solutionTrials.Take(optimizer.PopulationSize)) { - double[] x = trans.Transform(t.ParamsDict).ToArray(); + double[] x = trans.Transform(t.Params).ToArray(); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } @@ -287,7 +287,7 @@ public override OrderedDict InferRelativeSearchSpace(S { if (t.Value is null) continue; double[] x; - try { x = trans.Transform(t.ParamsDict).ToArray(); } + try { x = trans.Transform(t.Params).ToArray(); } catch (Exception) { continue; } if (x.Length != nDim) continue; pop.Add(x); diff --git a/src/OptunaSharp.Core/Samplers/EvolutionStrategy/EvolutionStrategySampler.cs b/src/OptunaSharp.Core/Samplers/EvolutionStrategy/EvolutionStrategySampler.cs index 27d0670..8e844d5 100644 --- a/src/OptunaSharp.Core/Samplers/EvolutionStrategy/EvolutionStrategySampler.cs +++ b/src/OptunaSharp.Core/Samplers/EvolutionStrategy/EvolutionStrategySampler.cs @@ -162,7 +162,7 @@ public override OrderedDict InferRelativeSearchSpace(S var solutions = new List<(double[] X, double Value)>(optimizer.PopulationSize); foreach (var t in solutionTrials.Take(optimizer.PopulationSize)) { - double[] x = trans.Transform(t.ParamsDict).ToArray(); + double[] x = trans.Transform(t.Params).ToArray(); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } diff --git a/src/OptunaSharp.Core/Samplers/GPSampler.cs b/src/OptunaSharp.Core/Samplers/GPSampler.cs index 68b4aa4..c702e90 100644 --- a/src/OptunaSharp.Core/Samplers/GPSampler.cs +++ b/src/OptunaSharp.Core/Samplers/GPSampler.cs @@ -187,7 +187,7 @@ public override OrderedDict InferRelativeSearchSpace(S { var internalSearchSpace = new GpSearchSpace(searchSpace); var completedParams = completedTrials - .Select(t => (IReadOnlyDictionary)t.ParamsDict) + .Select(t => t.Params) .ToList(); double[,] normalizedParams = internalSearchSpace.GetNormalizedParams(completedParams); @@ -641,8 +641,10 @@ public override void AfterTrial(Study study, FrozenTrial trial, TrialState state /// The adaptation coefficients for the current covariance rank. private static IReadOnlyDictionary GetParams(FrozenTrial trial) { + // Read-only throughout: a finished record may be compact (plan 13, item S12), and the mutable accessor would + // give it its dictionary back on every ask. if (trial.State.IsFinished()) - return trial.ParamsDict; + return trial.Params; var parts = new List(); int i = 0; @@ -653,10 +655,10 @@ public override void AfterTrial(Study study, FrozenTrial trial, TrialState state i++; } if (parts.Count == 0) - return trial.ParamsDict; + return trial.Params; var merged = ParseJsonObject(string.Concat(parts)); - foreach (var kv in trial.ParamsDict) + foreach (var kv in trial.Params) merged[kv.Key] = kv.Value; return merged; } diff --git a/src/OptunaSharp.Core/Samplers/Janus/JanusMooCmix.cs b/src/OptunaSharp.Core/Samplers/Janus/JanusMooCmix.cs index 10b43d3..200c8bc 100644 --- a/src/OptunaSharp.Core/Samplers/Janus/JanusMooCmix.cs +++ b/src/OptunaSharp.Core/Samplers/Janus/JanusMooCmix.cs @@ -157,9 +157,9 @@ internal JanusMooCmix( for (int ti = start; ti < completed.Count; ti++) { var t = completed[ti]; - if (!paramNames.All(n => t.ParamsDict.ContainsKey(n))) continue; + if (!paramNames.All(n => t.Params.ContainsKey(n))) continue; double[] row; - try { row = paramNames.Select(n => ToNorm(t.ParamsDict[n], transforms[n])).ToArray(); } + try { row = paramNames.Select(n => ToNorm(t.Params[n], transforms[n])).ToArray(); } catch (Exception) { continue; } rows.Add(row); usedTrials.Add(t); @@ -310,7 +310,7 @@ private double ScalarValue(FrozenTrial trial) /// The selected normalized center, or null when no compatible trial exists. private double[]? ChooseCenter(List trials, List paramNames, Dictionary transforms) { - var complete = trials.Where(t => paramNames.All(n => t.ParamsDict.ContainsKey(n))).ToList(); + var complete = trials.Where(t => paramNames.All(n => t.Params.ContainsKey(n))).ToList(); if (complete.Count == 0) return null; var valueRows = complete.Where(t => t.Values is not null).Select(t => t.Values!.ToArray()).ToList(); @@ -348,7 +348,7 @@ private double ScalarValue(FrozenTrial trial) int d = paramNames.Count; var center = new double[d]; foreach (var t in top) - for (int j = 0; j < d; j++) center[j] += ToNorm(t.ParamsDict[paramNames[j]], transforms[paramNames[j]]); + for (int j = 0; j < d; j++) center[j] += ToNorm(t.Params[paramNames[j]], transforms[paramNames[j]]); for (int j = 0; j < d; j++) center[j] /= top.Count; return center; } diff --git a/src/OptunaSharp.Core/Samplers/Janus/JanusOptimizer.cs b/src/OptunaSharp.Core/Samplers/Janus/JanusOptimizer.cs index 1363504..7b3773e 100644 --- a/src/OptunaSharp.Core/Samplers/Janus/JanusOptimizer.cs +++ b/src/OptunaSharp.Core/Samplers/Janus/JanusOptimizer.cs @@ -196,7 +196,7 @@ private void Harvest() double[] x; try { - x = _trans.Transform(t.ParamsDict).ToArray(); + x = _trans.Transform(t.Params).ToArray(); } catch (Exception) { @@ -943,7 +943,7 @@ internal void UpdateEs() for (int k = 0; k < _popsize; k++) { var t = ready[b * _popsize + k]; - double[] x = _trans.Transform(t.ParamsDict).ToArray(); + double[] x = _trans.Transform(t.Params).ToArray(); double y = _direction == StudyDirection.Maximize ? -t.Value!.Value : t.Value!.Value; solutions.Add((x, y)); _trialsUsed.Add(t.Number); diff --git a/src/OptunaSharp.Core/Samplers/LmCmaSampler.cs b/src/OptunaSharp.Core/Samplers/LmCmaSampler.cs index b9212c0..4f9c47a 100644 --- a/src/OptunaSharp.Core/Samplers/LmCmaSampler.cs +++ b/src/OptunaSharp.Core/Samplers/LmCmaSampler.cs @@ -194,7 +194,7 @@ public override OrderedDict InferRelativeSearchSpace(S { // Re-derive x in optimizer space from the stored (clamped) params, so the optimizer learns // from the actually-evaluated points. Flip sign for maximization. - double[] x = trans.Transform(t.ParamsDict).ToArray(); + double[] x = trans.Transform(t.Params).ToArray(); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } @@ -379,11 +379,11 @@ private List GetTrials(Study study) completeTrials.Add(t); } else if (t.State == TrialState.Pruned - && t.IntermediateValues.Count > 0 + && t.IntermediateValuesForRead.Count > 0 && _considerPrunedTrials) { - long maxStep = t.IntermediateValues.Keys.Max(); - double value = t.IntermediateValues[maxStep]; + long maxStep = t.IntermediateValuesForRead.Keys.Max(); + double value = t.IntermediateValuesForRead[maxStep]; var copied = t.DeepCopy(); copied.Value = value; completeTrials.Add(copied); diff --git a/src/OptunaSharp.Core/Samplers/Moead/MoeadOffspring.cs b/src/OptunaSharp.Core/Samplers/Moead/MoeadOffspring.cs index cf0e39f..6669a18 100644 --- a/src/OptunaSharp.Core/Samplers/Moead/MoeadOffspring.cs +++ b/src/OptunaSharp.Core/Samplers/Moead/MoeadOffspring.cs @@ -65,8 +65,8 @@ internal static class MoeadOffspring if (categoricalSearchSpace.Count > 0) { var catKeys = categoricalSearchSpace.Keys.ToList(); - var parent0Cat = catKeys.Select(p => parents[0].ParamsDict[p]).ToArray(); - var parentLastCat = catKeys.Select(p => parents[^1].ParamsDict[p]).ToArray(); + var parent0Cat = catKeys.Select(p => parents[0].Params[p]).ToArray(); + var parentLastCat = catKeys.Select(p => parents[^1].Params[p]).ToArray(); var r = rng.rand(catKeys.Count).ToArray(); for (int j = 0; j < catKeys.Count; j++) childParams[catKeys[j]] = r[j] >= swappingProb ? parentLastCat[j] : parent0Cat[j]; @@ -83,7 +83,7 @@ internal static class MoeadOffspring { var row = new OrderedDict(); foreach (var k in numKeys) - row[k] = parents[i].ParamsDict[k]; + row[k] = parents[i].Params[k]; parentsNumerical[i] = transform.Transform(row).ToArray(); } diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs index 32e0c34..76acc38 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs @@ -94,7 +94,7 @@ public NSGAIIChildGenerationStrategy( else { int parentPopulationSize = parentPopulation.Count; - var parentParams = parentPopulation[(int)_rng.Rng.randint(0, parentPopulationSize)].ParamsDict; + var parentParams = parentPopulation[(int)_rng.Rng.randint(0, parentPopulationSize)].Params; childParams = new OrderedDict(); foreach (var name in searchSpace.Keys) childParams[name] = parentParams[name]; diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs index f485a18..c669d3f 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs @@ -87,8 +87,8 @@ internal static class NsgaiiCrossover { var catKeys = categoricalSearchSpace.Keys.ToList(); // parents_categorical_params = [[parents[0][p]...], [parents[-1][p]...]] (2 rows). - var parent0Cat = catKeys.Select(p => parents[0].ParamsDict[p]).ToArray(); - var parentLastCat = catKeys.Select(p => parents[^1].ParamsDict[p]).ToArray(); + var parent0Cat = catKeys.Select(p => parents[0].Params[p]).ToArray(); + var parentLastCat = catKeys.Select(p => parents[^1].Params[p]).ToArray(); var childCategorical = InlinedCategoricalUniformCrossover( parent0Cat, parentLastCat, rng, swappingProb, catKeys.Count); @@ -106,7 +106,7 @@ internal static class NsgaiiCrossover { var row = new OrderedDict(); foreach (var k in numKeys) - row[k] = parents[i].ParamsDict[k]; + row[k] = parents[i].Params[k]; parentsNumerical[i] = numericalTransform.Transform(row).ToArray(); } diff --git a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/ConditionalParams.cs b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/ConditionalParams.cs index 3181cbc..3479072 100644 --- a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/ConditionalParams.cs +++ b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/ConditionalParams.cs @@ -378,10 +378,10 @@ private sealed record LatentEntry( { foreach (var name in _rules.Keys) { - if (trial.ParamsDict.TryGetValue(name, out var active)) + if (trial.Params.TryGetValue(name, out var active)) latent[name] = active; } - if (trial.SystemAttrsDict.TryGetValue(DormantAttrKey, out var raw) && raw is not null) + if (trial.SystemAttrs.TryGetValue(DormantAttrKey, out var raw) && raw is not null) { foreach (var (name, value) in CoerceDormant(raw)) { @@ -496,8 +496,8 @@ private bool ShouldKeep( { foreach (var name in Order) { - bool present = trial.ParamsDict.ContainsKey(name); - bool active = IsActive(name, trial.ParamsDict); + bool present = trial.Params.ContainsKey(name); + bool active = IsActive(name, trial.Params); if (present == active) continue; yield return (name, present diff --git a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptChildGenerationStrategy.cs b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptChildGenerationStrategy.cs index 10c706d..52df591 100644 --- a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptChildGenerationStrategy.cs +++ b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptChildGenerationStrategy.cs @@ -110,7 +110,7 @@ public NsgaiiiCptChildGenerationStrategy( else { int parentPopulationSize = parentPopulation.Count; - var parentParams = parentPopulation[(int)_rng.Rng.randint(0, parentPopulationSize)].ParamsDict; + var parentParams = parentPopulation[(int)_rng.Rng.randint(0, parentPopulationSize)].Params; childParams = new OrderedDict(); if (_conditionalParams is null) { diff --git a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptCrossover.cs b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptCrossover.cs index 2b18507..5b03ad5 100644 --- a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptCrossover.cs @@ -116,8 +116,8 @@ internal static class NsgaiiiCptCrossover if (categoricalSearchSpace.Count > 0) { - var first = parents[0].ParamsDict; - var last = parents[^1].ParamsDict; + var first = parents[0].Params; + var last = parents[^1].Params; var shared = categoricalSearchSpace.Keys.Where(k => first.ContainsKey(k) && last.ContainsKey(k)).ToList(); var crossed = shared.Count == 0 ? Array.Empty() @@ -150,7 +150,7 @@ internal static class NsgaiiiCptCrossover // so letting it consume RNG would make deactivation perturb the other parameters' math. if (conditionalParams.ResolveActive(key, childParams) == false) continue; - var carriers = parents.Where(p => p.ParamsDict.ContainsKey(key)).ToList(); + var carriers = parents.Where(p => p.Params.ContainsKey(key)).ToList(); if (carriers.Count == parents.Count) common[key] = distribution; else if (carriers.Count > 0) @@ -166,7 +166,7 @@ internal static class NsgaiiiCptCrossover { var row = new OrderedDict(); foreach (var key in common.Keys) - row[key] = parents[i].ParamsDict[key]; + row[key] = parents[i].Params[key]; parentsNumerical[i] = transform.Transform(row).ToArray(); } @@ -185,7 +185,7 @@ internal static class NsgaiiiCptCrossover else if (partialCarriers.TryGetValue(key, out var carriers)) { var carrier = carriers.Count == 1 ? carriers[0] : carriers[(int)rng.randint(0, carriers.Count)]; - childParams[key] = carrier.ParamsDict[key]; + childParams[key] = carrier.Params[key]; } } diff --git a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptSearchSpace.cs b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptSearchSpace.cs index 47a4cdb..4fd4811 100644 --- a/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptSearchSpace.cs +++ b/src/OptunaSharp.Core/Samplers/NsgaiiiCpt/NsgaiiiCptSearchSpace.cs @@ -182,14 +182,14 @@ internal static (OrderedDict? SearchSpace, int NextCac if (searchSpace is null) { - searchSpace = new OrderedDict(trial.DistributionsDict); + searchSpace = new OrderedDict(trial.Distributions); continue; } var next = new OrderedDict(); foreach (var (name, distribution) in searchSpace) { - if (trial.DistributionsDict.TryGetValue(name, out var trialDist)) + if (trial.Distributions.TryGetValue(name, out var trialDist)) { if (trialDist.Equals(distribution)) next[name] = distribution; @@ -201,7 +201,7 @@ internal static (OrderedDict? SearchSpace, int NextCac next[name] = distribution; // Absent because inactive — the expected case. } } - foreach (var (name, distribution) in trial.DistributionsDict) + foreach (var (name, distribution) in trial.Distributions) { if (!conditionalParams.IsConditional(name) || next.ContainsKey(name) || excluded.Contains(name)) continue; diff --git a/src/OptunaSharp.Core/Samplers/PsoSampler.cs b/src/OptunaSharp.Core/Samplers/PsoSampler.cs index 8b649af..06edca5 100644 --- a/src/OptunaSharp.Core/Samplers/PsoSampler.cs +++ b/src/OptunaSharp.Core/Samplers/PsoSampler.cs @@ -213,7 +213,7 @@ public override OrderedDict InferRelativeSearchSpace(S var solutions = new List<(double[] X, double Value)>(optimizer.PopulationSize); foreach (var t in solutionTrials.Take(optimizer.PopulationSize)) { - double[] x = trans.Transform(t.ParamsDict).ToArray(); + double[] x = trans.Transform(t.Params).ToArray(); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } @@ -295,7 +295,7 @@ public override OrderedDict InferRelativeSearchSpace(S { if (t.Value is null) continue; double[] x; - try { x = trans.Transform(t.ParamsDict).ToArray(); } + try { x = trans.Transform(t.Params).ToArray(); } catch (Exception) { continue; } if (x.Length != nDim) continue; pop.Add(x); diff --git a/src/OptunaSharp.Core/Samplers/QMCSampler.cs b/src/OptunaSharp.Core/Samplers/QMCSampler.cs index ce18943..457ea15 100644 --- a/src/OptunaSharp.Core/Samplers/QMCSampler.cs +++ b/src/OptunaSharp.Core/Samplers/QMCSampler.cs @@ -129,7 +129,7 @@ public override OrderedDict InferRelativeSearchSpace(S HashSet? intersectionKeys = null; foreach (var t in pendingTrials) { - var space = t.DistributionsDict; + var space = t.Distributions; foreach (var (name, dist) in space) unionSearchSpace[name] = dist; if (intersectionKeys is null) @@ -147,7 +147,7 @@ public override OrderedDict InferRelativeSearchSpace(S // If an initial trial was already made, construct the search space from the first trial. var result = new OrderedDict(); - foreach (var (name, dist) in firstTrial.DistributionsDict) + foreach (var (name, dist) in firstTrial.Distributions) result[name] = dist; return result; } diff --git a/src/OptunaSharp.Core/Samplers/RandomEmbeddingSampler.cs b/src/OptunaSharp.Core/Samplers/RandomEmbeddingSampler.cs index a7722cb..8a257cf 100644 --- a/src/OptunaSharp.Core/Samplers/RandomEmbeddingSampler.cs +++ b/src/OptunaSharp.Core/Samplers/RandomEmbeddingSampler.cs @@ -535,7 +535,7 @@ private double[] ReadLatent(Trial ask) private bool TryReadStoredLatent(FrozenTrial trial, out double[] y) { y = Array.Empty(); - if (!trial.SystemAttrsDict.TryGetValue(LatentAttrKey, out object? raw) || raw is not IEnumerable seq) + if (!trial.SystemAttrs.TryGetValue(LatentAttrKey, out object? raw) || raw is not IEnumerable seq) return false; var values = new List(_embedding.Dim); diff --git a/src/OptunaSharp.Core/Samplers/SaasboSampler.cs b/src/OptunaSharp.Core/Samplers/SaasboSampler.cs index 09802fc..4afa277 100644 --- a/src/OptunaSharp.Core/Samplers/SaasboSampler.cs +++ b/src/OptunaSharp.Core/Samplers/SaasboSampler.cs @@ -258,7 +258,7 @@ public override OrderedDict InferRelativeSearchSpace(S // ---- Bayesian-optimization phase: fit the SAAS GP and maximize the averaged EI ---- int n = completed.Count; double[,] normalizedParams = internalSpace.GetNormalizedParams( - completed.Select(t => (IReadOnlyDictionary)t.ParamsDict).ToList()); + completed.Select(t => t.Params).ToList()); double sign = study.Direction == StudyDirection.Minimize ? -1.0 : 1.0; diff --git a/src/OptunaSharp.Core/Samplers/TPESampler.cs b/src/OptunaSharp.Core/Samplers/TPESampler.cs index 2fa1a5b..9a48079 100644 --- a/src/OptunaSharp.Core/Samplers/TPESampler.cs +++ b/src/OptunaSharp.Core/Samplers/TPESampler.cs @@ -351,7 +351,7 @@ public override OrderedDict InferRelativeSearchSpace(S if (_warnIndependentSampling && IsMultivariate(study)) { // Avoid independent warning at the first sampling of `param_name`. - if (trials.Any(t => t.ParamsDict.ContainsKey(paramName))) + if (trials.Any(t => t.Params.ContainsKey(paramName))) _logger.Warning(string.Format( IndependentSamplingWarningTemplate, paramName, @@ -370,14 +370,18 @@ public override OrderedDict InferRelativeSearchSpace(S /// Trial snapshot being examined; copy its contents before modifying them. /// Study supplying trial history, objective directions, and storage. /// Recorded external parameters supplemented with readable shared relative-sampling attributes. - private OrderedDict GetParams(FrozenTrial trial, Study study) + /// + /// Read-only throughout (plan 13, item S12): it runs for every trial on every ask, and a finished record of a + /// storage built with compactFinishedTrials would get its dictionary back from the mutable accessor. + /// + private IReadOnlyDictionary GetParams(FrozenTrial trial, Study study) { if (trial.State.IsFinished() || !IsMultivariate(study)) - return trial.ParamsDict; + return trial.Params; var paramsStrs = new List(); int i = 0; - while (trial.SystemAttrsDict.TryGetValue($"{RelativeParamsKey}:{i}", out var chunk) + while (trial.SystemAttrs.TryGetValue($"{RelativeParamsKey}:{i}", out var chunk) && chunk is string s && s.Length > 0) { paramsStrs.Add(s); @@ -385,7 +389,7 @@ public override OrderedDict InferRelativeSearchSpace(S } if (paramsStrs.Count == 0) - return trial.ParamsDict; + return trial.Params; OrderedDict @params; try @@ -395,9 +399,9 @@ public override OrderedDict InferRelativeSearchSpace(S catch { // Incomplete JSON from a concurrent write: fall back to trial.params. - return trial.ParamsDict; + return trial.Params; } - foreach (var (k, v) in trial.ParamsDict) + foreach (var (k, v) in trial.Params) @params[k] = v; return @params; } @@ -428,9 +432,34 @@ private OrderedDict GetInternalRepr( } int rows = 0; + // A compact finished trial (plan 13, item S12; InMemoryStorage built with compactFinishedTrials) stores each + // value's internal form, exactly what ToInternalRepr would rebuild from its external one, so a parameter whose + // search-space distribution converts as the trial's does is copied without boxing and converting it back. The + // positions depend only on the trial's schema, which a static space shares across the whole history. + ParamSchema? schema = null; + int[] positions = Array.Empty(); foreach (var trial in trials) { var @params = GetParams(trial, study); + if (@params is CompactParams compact) + { + if (!ReferenceEquals(compact.Schema, schema)) + { + schema = compact.Schema; + positions = schema.PositionsFor(names, distributions); + } + if (Array.IndexOf(positions, ParamSchema.Absent) >= 0) + continue; + for (int i = 0; i < nParams; i++) + { + columns[i][rows] = positions[i] >= 0 + ? compact.InternalAt(positions[i]) + : distributions[i].ToInternalRepr(@params[names[i]]); + } + rows++; + continue; + } + // search_space.keys() <= params.keys(): the trial must carry every search-space param. bool carriesAll = true; for (int i = 0; i < nParams && carriesAll; i++) diff --git a/src/OptunaSharp.Core/Samplers/TurboSampler.cs b/src/OptunaSharp.Core/Samplers/TurboSampler.cs index 95e7240..203ab12 100644 --- a/src/OptunaSharp.Core/Samplers/TurboSampler.cs +++ b/src/OptunaSharp.Core/Samplers/TurboSampler.cs @@ -314,7 +314,7 @@ public override OrderedDict InferRelativeSearchSpace(S int m = restartTrials.Count; double[,] normalizedParams = internalSpace.GetNormalizedParams( - restartTrials.Select(t => (IReadOnlyDictionary)t.ParamsDict).ToList()); + restartTrials.Select(t => t.Params).ToList()); // Standardize the local run's scores for the GP fit (the paper standardizes objective values before // fitting; §C). `clipped` keeps the maximization-oriented observations for the center rule. @@ -485,7 +485,7 @@ private TrustRegionSample BuildRegionSample( int mRestart = restartTrials.Count; double[,] normalizedParams = internalSpace.GetNormalizedParams( - restartTrials.Select(t => (IReadOnlyDictionary)t.ParamsDict).ToList()); + restartTrials.Select(t => t.Params).ToList()); // Standardize the run's maximization-oriented scores for the GP fit; keep mean/std to un-standardize the // Thompson draw for the cross-region comparison (each region standardizes by its OWN constants). diff --git a/src/OptunaSharp.Core/Samplers/VkdCmaSampler.cs b/src/OptunaSharp.Core/Samplers/VkdCmaSampler.cs index eb2a372..4c9df11 100644 --- a/src/OptunaSharp.Core/Samplers/VkdCmaSampler.cs +++ b/src/OptunaSharp.Core/Samplers/VkdCmaSampler.cs @@ -232,7 +232,7 @@ public override OrderedDict InferRelativeSearchSpace(S { // Re-derive x in optimizer space from the stored (clamped) params, so the optimizer learns // from the actually-evaluated points. Flip sign for maximization. - double[] x = trans.Transform(t.ParamsDict).ToArray(); + double[] x = trans.Transform(t.Params).ToArray(); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } @@ -395,11 +395,11 @@ private List GetTrials(Study study) completeTrials.Add(t); } else if (t.State == TrialState.Pruned - && t.IntermediateValues.Count > 0 + && t.IntermediateValuesForRead.Count > 0 && _considerPrunedTrials) { - long maxStep = t.IntermediateValues.Keys.Max(); - double value = t.IntermediateValues[maxStep]; + long maxStep = t.IntermediateValuesForRead.Keys.Max(); + double value = t.IntermediateValuesForRead[maxStep]; var copied = t.DeepCopy(); copied.Value = value; completeTrials.Add(copied); diff --git a/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs b/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs index ff8172e..4d91720 100644 --- a/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs +++ b/src/OptunaSharp.Core/SearchSpace/GroupDecomposedSearchSpace.cs @@ -15,8 +15,11 @@ internal sealed class SearchSpaceGroup public List> SearchSpaces => _searchSpaces; /// Splits existing groups so every resulting group has the same presence pattern in the incoming trial. - /// The parameter distributions being incorporated into the decomposed search space. - public void AddDistributions(OrderedDict distributions) + /// + /// The parameter distributions being incorporated into the decomposed search space; only read, so a compact + /// finished trial's read-only view (plan 13, item S12) serves as well as a dictionary. + /// + public void AddDistributions(IReadOnlyDictionary distributions) { var distKeys = new HashSet(distributions.Keys); var nextSearchSpaces = new List>(); @@ -104,7 +107,7 @@ public SearchSpaceGroup Calculate(Study study, bool useCache = false) foreach (var trial in study.GetTrialsForSampler(useCache)) { if (System.Array.IndexOf(statesOfInterest, trial.State) >= 0) - _searchSpace.AddDistributions(trial.DistributionsDict); + _searchSpace.AddDistributions(trial.Distributions); } return _searchSpace.DeepCopy(); diff --git a/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs b/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs index 661038d..cbf5385 100644 --- a/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs +++ b/src/OptunaSharp.Core/SearchSpace/IntersectionSearchSpace.cs @@ -58,14 +58,14 @@ internal static (OrderedDict? SearchSpace, int NextCac if (searchSpace is null) { - searchSpace = new OrderedDict(trial.DistributionsDict); + searchSpace = new OrderedDict(trial.Distributions); continue; } var next = new OrderedDict(); foreach (var (name, distribution) in searchSpace) { - if (trial.DistributionsDict.TryGetValue(name, out var trialDist) && trialDist.Equals(distribution)) + if (trial.Distributions.TryGetValue(name, out var trialDist) && trialDist.Equals(distribution)) next[name] = distribution; } searchSpace = next; diff --git a/src/OptunaSharp.Core/Storages/Callbacks.cs b/src/OptunaSharp.Core/Storages/Callbacks.cs index daca9a1..fe43834 100644 --- a/src/OptunaSharp.Core/Storages/Callbacks.cs +++ b/src/OptunaSharp.Core/Storages/Callbacks.cs @@ -52,7 +52,7 @@ public void Invoke(Study study, FrozenTrial trial) ["failed_trial"] = (long)trial.Number, ["retry_history"] = new List(), }; - foreach (var (key, value) in trial.SystemAttrsDict) + foreach (var (key, value) in trial.SystemAttrs) systemAttrs[key] = value; var retryHistory = (List)systemAttrs["retry_history"]!; @@ -85,7 +85,7 @@ public static implicit operator Action(RetryHeartbeatStaleTr /// The retry marker exceeds Int32. /// The retry marker is not convertible to an integer. public static int? RetriedTrialNumber(FrozenTrial trial) - => trial.SystemAttrsDict.GetValueOrDefault("failed_trial") is { } value + => trial.SystemAttrs.GetValueOrDefault("failed_trial") is { } value ? Convert.ToInt32(value, System.Globalization.CultureInfo.InvariantCulture) : null; @@ -97,7 +97,7 @@ public static implicit operator Action(RetryHeartbeatStaleTr /// Trial record or active trial on which the operation acts. /// A new list of ancestor trial numbers in retry order; empty when absent. public static List RetryHistory(FrozenTrial trial) - => trial.SystemAttrsDict.GetValueOrDefault("retry_history") is List history + => trial.SystemAttrs.GetValueOrDefault("retry_history") is List history ? history.Select(n => Convert.ToInt32(n, System.Globalization.CultureInfo.InvariantCulture)).ToList() : new List(); } diff --git a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs index cb73637..d95871b 100644 --- a/src/OptunaSharp.Core/Storages/InMemoryStorage.cs +++ b/src/OptunaSharp.Core/Storages/InMemoryStorage.cs @@ -33,6 +33,48 @@ public class InMemoryStorage : BaseStorage private readonly object _lock = new(); private readonly Dictionary _prevWaitingTrialNumber = new(); + private readonly bool _compactFinishedTrials; + + /// Creates an empty storage that keeps every trial's dictionaries as Optuna's in-memory storage does. + public InMemoryStorage() + : this(compactFinishedTrials: false) + { + } + + /// + /// Creates an empty storage, optionally keeping finished trials in a compact form (Rustuna plan 13, item S12). + /// + /// + /// When true, each trial is compacted as it finishes: parameters become one internal value per name against a + /// parameter schema the study's trials share, distributions come from that schema, attributes become arrays + /// with long ASCII strings stored as bytes, and empty dictionaries take no memory. False (the default) keeps + /// Optuna's representation exactly. + /// + /// + /// + /// What stays the same: every value any API returns, and so every sampler's results; a seeded study gives + /// the same trials either way. A finished 40-parameter random-search trial took 6.7 KB in memory and takes + /// 0.7 KB compact; a CMA-ES trial 13.5 KB and 2.7 KB, its optimizer states stored as bytes (100,000 trials: + /// 1,287 MB and 257 MB). Run times stay the same within noise. + /// + /// + /// What changes, which is why it is opt-in: a finished trial's , + /// , and + /// become read-only views rather than + /// objects (a cast to OrderedDict fails, a write through IDictionary throws), and numeric values + /// and long strings are rebuilt on each read, so two reads are equal but not the same object. The mutable + /// accessors ( and the other …Dict properties, and + /// ) still return real dictionaries: the first such read gives + /// the trial back a dictionary of its own, which it keeps, so changes made through them stay, at the cost of + /// that trial's saving. A finished trial no longer shares its dictionaries with the running record it was + /// copied from. A read of a numeric parameter through the views allocates a box; TPE, which reads every + /// trial on every ask, copies the stored internal values instead. + /// + /// + public InMemoryStorage(bool compactFinishedTrials) => _compactFinishedTrials = compactFinishedTrials; + + /// Whether this storage keeps finished trials in the compact form (see the constructor). + public bool CompactFinishedTrials => _compactFinishedTrials; /// Objective directions in the same order as the objective values. /// Name identifying the study in the selected storage. @@ -258,6 +300,10 @@ public override int CreateNewTrial(int studyId, FrozenTrial? templateTrial = nul // Both branches built a record with dictionaries nobody else holds, so its writes may go in place until a // reader receives it (see FrozenTrial.StorageExclusive). trial.StorageExclusive = true; + // A trial added already finished (Study.AddTrial) is compacted as one finishing here would be; it is never + // written again, so the exclusive flag above stays unused for it. + if (_compactFinishedTrials && trial.State.IsFinished()) + trial.CompactFinished(SchemasOf(studyId)); _trialIdToStudyIdAndNumber[trialId] = (studyId, trial.Number); _studies[studyId].Trials.Add(trial); UpdateCache(trialId, studyId); @@ -360,6 +406,15 @@ private FrozenTrial WritableTrialUnlocked(int trialId) return own; } + /// The study's schema cache for compact finished trials, created on first use (plan 13, item S12). + /// A study of this storage; the caller holds the lock. + /// The cache the study's compact trials share schemas through. + private ParamSchemaCache SchemasOf(int studyId) + { + StudyInfo study = _studies[studyId]; + return study.Schemas ??= new ParamSchemaCache(); + } + /// /// How many writes had to copy a trial's record first because a reader may hold it (Rustuna plan 13, item S4): /// a diagnostic the tests read to prove that suggestions nobody reads between are written in place. @@ -442,6 +497,12 @@ public override double GetTrialParam(int trialId, string paramName) { var trial = GetTrialUnlocked(trialId); + // A compact finished trial (plan 13, item S12) stores exactly the internal value the conversion below would + // rebuild through its own distribution, so it is returned as stored. A missing name falls through to the + // dictionary path's KeyNotFoundException. + if (trial.Params is CompactParams compact && compact.TryGetInternal(paramName, out double stored)) + return stored; + var distribution = trial.Distributions[paramName]; return distribution.ToInternalRepr(trial.Params[paramName]); } @@ -473,8 +534,12 @@ public override bool SetTrialStateValues(int trialId, TrialState state, IReadOnl if (state.IsFinished()) { trial.DatetimeComplete = DateTime.Now; - SetTrial(trialId, trial); int studyId = _trialIdToStudyIdAndNumber[trialId].StudyId; + // Compact the new record before it is published: no reader can see this object yet, and the + // dictionaries it shares with the record it replaces are only read (FrozenTrial.CompactFinished). + if (_compactFinishedTrials) + trial.CompactFinished(SchemasOf(studyId)); + SetTrial(trialId, trial); UpdateCache(trialId, studyId); } else @@ -790,5 +855,11 @@ internal StudyInfo(string name, List directions) internal List Directions { get; } /// Identifier of the best completed trial cached for single-objective lookup. internal int? BestTrialId { get; set; } + + /// + /// The study's parameter schemas for compact finished trials (plan 13, item S12); created by the first trial + /// that finishes in a storage built with compactFinishedTrials: true, so other storages never hold one. + /// + internal ParamSchemaCache? Schemas { get; set; } } } diff --git a/src/OptunaSharp.Core/Study.cs b/src/OptunaSharp.Core/Study.cs index 185c15f..fcdb7de 100644 --- a/src/OptunaSharp.Core/Study.cs +++ b/src/OptunaSharp.Core/Study.cs @@ -854,9 +854,11 @@ internal bool ShouldSkipEnqueue(OrderedDict @params) { foreach (var trial in GetTrials(deepcopy: false)) { + // The read-only accessors: this scans every trial, and the mutable ones would give each compact finished + // record (plan 13, item S12) its dictionaries back. IReadOnlyDictionary trialParams = - trial.SystemAttrsDict.GetValueOrDefault("fixed_params") as OrderedDict - ?? (IReadOnlyDictionary)trial.ParamsDict; + trial.SystemAttrs.GetValueOrDefault("fixed_params") as OrderedDict + ?? trial.Params; if (!new HashSet(trialParams.Keys).SetEquals(@params.Keys)) { diff --git a/src/OptunaSharp.Core/Terminator/EmmrEvaluator.cs b/src/OptunaSharp.Core/Terminator/EmmrEvaluator.cs index 3ffcf25..c768a90 100644 --- a/src/OptunaSharp.Core/Terminator/EmmrEvaluator.cs +++ b/src/OptunaSharp.Core/Terminator/EmmrEvaluator.cs @@ -82,7 +82,7 @@ public override double Evaluate(IReadOnlyList trials, StudyDirectio var searchSpace = new GpSearchSpace(optunaSearchSpace); double[,] normalizedParams = searchSpace.GetNormalizedParams( - completeTrials.Select(t => (IReadOnlyDictionary)t.ParamsDict).ToList()); + completeTrials.Select(t => t.Params).ToList()); if (searchSpace.Dim == 0) { OptunaWarnings.Warn( diff --git a/src/OptunaSharp.Core/Terminator/ImprovementEvaluator.cs b/src/OptunaSharp.Core/Terminator/ImprovementEvaluator.cs index b1ef490..49d334d 100644 --- a/src/OptunaSharp.Core/Terminator/ImprovementEvaluator.cs +++ b/src/OptunaSharp.Core/Terminator/ImprovementEvaluator.cs @@ -138,7 +138,7 @@ public override double Evaluate(IReadOnlyList trials, StudyDirectio var searchSpace = new GpSearchSpace(optunaSearchSpace); double[,] normalizedParams = searchSpace.GetNormalizedParams( - completeTrials.Select(t => (IReadOnlyDictionary)t.ParamsDict).ToList()); + completeTrials.Select(t => t.Params).ToList()); (double[,] normalizedTopNParams, double[] topNValues) = GetTopN(normalizedParams, values); double topNValuesMean = MathUtil.Mean(topNValues); diff --git a/src/OptunaSharp.Core/Visualization/IntermediateValues.cs b/src/OptunaSharp.Core/Visualization/IntermediateValues.cs index af0cfae..356351f 100644 --- a/src/OptunaSharp.Core/Visualization/IntermediateValues.cs +++ b/src/OptunaSharp.Core/Visualization/IntermediateValues.cs @@ -45,10 +45,10 @@ static bool SatisfiesConstraints(FrozenTrial trial) => trial.Constraints.Values.All(x => x <= 0.0); var trialInfos = trials - .Where(trial => trial.IntermediateValues.Count > 0) + .Where(trial => trial.IntermediateValuesForRead.Count > 0) .Select(trial => new TrialInfo( trial.Number, - trial.IntermediateValues + trial.IntermediateValuesForRead .OrderBy(kv => kv.Key) .Select(kv => (kv.Key, kv.Value)) .ToList(), diff --git a/src/OptunaSharp.Core/Visualization/VisualizationUtils.cs b/src/OptunaSharp.Core/Visualization/VisualizationUtils.cs index d91d60a..0afbf78 100644 --- a/src/OptunaSharp.Core/Visualization/VisualizationUtils.cs +++ b/src/OptunaSharp.Core/Visualization/VisualizationUtils.cs @@ -173,7 +173,8 @@ internal static string MakeHovertext(FrozenTrial trial) { { "number", trial.Number }, { "values", trial.Values }, - { "params", trial.ParamsDict }, + // Read-only: a compact finished record (plan 13, item S12) serializes its view as the same JSON object. + { "params", trial.Params }, }; if (userAttrs.Count > 0) dict["user_attrs"] = userAttrs; diff --git a/src/OptunaSharp.Interop.BoTorch/BoTorchSampler.cs b/src/OptunaSharp.Interop.BoTorch/BoTorchSampler.cs index 78df4eb..615d574 100644 --- a/src/OptunaSharp.Interop.BoTorch/BoTorchSampler.cs +++ b/src/OptunaSharp.Interop.BoTorch/BoTorchSampler.cs @@ -186,7 +186,7 @@ public override OrderedDict InferRelativeSearchSpace(S for (int i = 0; i < nCompleted; i++) { FrozenTrial t = completedTrials[i]; - double[] row = trans.Transform(t.ParamsDict).ToArray(); + double[] row = trans.Transform(t.Params).ToArray(); for (int d = 0; d < nBounds; d++) completedParams[i, d] = row[d]; @@ -239,7 +239,7 @@ public override OrderedDict InferRelativeSearchSpace(S var rows = new List(); foreach (FrozenTrial t in runningTrials) if (searchSpace.Keys.All(k => t.Params.ContainsKey(k))) - rows.Add(trans.Transform(t.ParamsDict).ToArray()); + rows.Add(trans.Transform(t.Params).ToArray()); if (rows.Count > 0) { runningParams = new double[rows.Count, nBounds]; diff --git a/src/OptunaSharp.Interop.Pymoo/PymooSampler.cs b/src/OptunaSharp.Interop.Pymoo/PymooSampler.cs index 84bd394..5ee022e 100644 --- a/src/OptunaSharp.Interop.Pymoo/PymooSampler.cs +++ b/src/OptunaSharp.Interop.Pymoo/PymooSampler.cs @@ -329,7 +329,7 @@ internal void Reinitialize(IReadOnlyList complete) // so they slot straight into the (pop_size, n) initial-population matrix — no element loop. NDArray gen0 = np.zeros(new Shape(_popsize, _nBounds)); for (int k = 0; k < _popsize; k++) - gen0[k] = _trans.Transform(complete[k].ParamsDict); + gen0[k] = _trans.Transform(complete[k].Params); _algo.Dispose(); using PyObject sampling = PymooInterop.ToNumpy(gen0); diff --git a/test/OptunaSharp.Tests.Interop/CompactStorageParityTests.cs b/test/OptunaSharp.Tests.Interop/CompactStorageParityTests.cs new file mode 100644 index 0000000..0d4701d --- /dev/null +++ b/test/OptunaSharp.Tests.Interop/CompactStorageParityTests.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AwesomeAssertions; +using OptunaSharp.Distributions; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Interop; + +/// +/// Plan 13, item S12, against live Optuna: studies run on InMemoryStorage(compactFinishedTrials: true) give +/// Python's trials bit for bit (TPE to its documented erf tolerance), attributes included, while every finished +/// record stays compact through the run. The compact storage is an opt-in memory layout, so the samplers must not be +/// able to tell it from Optuna's. +/// +[TestClass] +public class CompactStorageParityTests : OptunaInteropTestBase +{ + /// The storage under test: Optuna's in-memory storage with finished trials kept compactly. + /// A new compacting storage. + private static InMemoryStorage CompactStorage() => new(compactFinishedTrials: true); + + /// Every finished record of the study is still compact after the run. + /// A study on a compacting storage. + private static void AssertRecordsStayedCompact(Study study) + { + var records = study.Storage.GetAllTrials(study.StudyId, deepcopy: false); + records.Should().NotBeEmpty(); + records.Where(r => r.State.IsFinished()).Should().OnlyContain(r => r.IsCompact, + "the sampler and the snapshot read finished records through the read-only accessors"); + } + + /// Runs the Python study and asserts bit-exact full-state parity with the compact C# one. + /// The finished OptunaSharp study on a compacting storage. + /// Python that creates and runs the same study as study. + private void AssertFullParity(Study csStudy, string pyCreateAndRun) + { + AssertRecordsStayedCompact(csStudy); + var csState = StudySnapshot.Capture(csStudy); + PyExec(pyCreateAndRun); + StudySnapshot.AssertParity(PyStudyState("study"), csState, DatetimeComparison.NullnessOnly); + AssertRecordsStayedCompact(csStudy); + } + + /// + /// Random search over every parameter kind, writing attributes the compact storage stores as bytes (a long ASCII + /// string, an odd-length lowercase hexadecimal string), a non-ASCII string it keeps, and a nested list: every + /// trial and attribute matches Python's bit for bit. + /// + /// Sampler seed. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(0)] + [DataRow(17)] + public void RandomWithAttributes_FullState_IsBitIdentical(int seed) + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy(storage: CompactStorage(), studyName: "compact", + sampler: new RandomSampler(seed: seed)); + csStudy.Optimize(t => + { + double x = t.SuggestFloat("x", -5, 5); + t.SuggestFloat("lx", 1e-3, 1e3, log: true); + t.SuggestFloat("sx", -1, 1, step: 0.25); + long n = t.SuggestInt("n", -3, 3); + t.SuggestInt("ln", 1, 64, log: true); + t.SuggestCategorical("c", new CategoricalChoice[] { "a", CategoricalChoice.None, true, 7L, 2.5 }); + t.SetUserAttr("blob", new string('q', 90) + t.Number); + t.SetUserAttr("hex", string.Concat(Enumerable.Repeat("0a1b2c3d4e5f", 6)) + "9"); + t.SetUserAttr("unicode", new string('é', 70)); + t.SetUserAttr("nested", new List { (long)t.Number, "two", 3.5, null }); + return x * x + n; + }, nTrials: 30); + + AssertFullParity(csStudy, + $"study = optuna.create_study(study_name='compact', sampler=optuna.samplers.RandomSampler(seed={seed}))\n" + + "def _obj(t):\n" + + " x = t.suggest_float('x', -5, 5)\n" + + " t.suggest_float('lx', 1e-3, 1e3, log=True)\n" + + " t.suggest_float('sx', -1, 1, step=0.25)\n" + + " n = t.suggest_int('n', -3, 3)\n" + + " t.suggest_int('ln', 1, 64, log=True)\n" + + " t.suggest_categorical('c', ['a', None, True, 7, 2.5])\n" + + " t.set_user_attr('blob', 'q' * 90 + str(t.number))\n" + + " t.set_user_attr('hex', '0a1b2c3d4e5f' * 6 + '9')\n" + + " t.set_user_attr('unicode', '\\u00e9' * 70)\n" + + " t.set_user_attr('nested', [t.number, 'two', 3.5, None])\n" + + " return x * x + n\n" + + "study.optimize(_obj, n_trials=30)\n"); + } + + /// NSGA-II over a mixed space reads its parents from compact records and matches Python bit for bit. + /// Sampler seed. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(0)] + [DataRow(42)] + public void NsgaiiMixedSpace_FullState_IsBitIdentical(int seed) + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy(storage: CompactStorage(), studyName: "compact", + sampler: new NSGAIISampler(populationSize: 5, seed: seed), direction: StudyDirection.Minimize); + csStudy.Optimize(t => + { + double x = t.SuggestFloat("x", -5, 5); + long y = t.SuggestInt("y", 0, 10); + t.SuggestCategorical("z", new CategoricalChoice[] { "a", "b", "c" }); + double lx = t.SuggestFloat("lx", 1e-3, 1e3, log: true); + return Math.Pow(x - 2, 2) + y + lx; + }, nTrials: 24); + + AssertFullParity(csStudy, + "study = optuna.create_study(study_name='compact', direction='minimize', " + + $"sampler=optuna.samplers.NSGAIISampler(population_size=5, seed={seed}))\n" + + "def _obj(t):\n" + + " x = t.suggest_float('x', -5, 5)\n" + + " y = t.suggest_int('y', 0, 10)\n" + + " t.suggest_categorical('z', ['a', 'b', 'c'])\n" + + " lx = t.suggest_float('lx', 1e-3, 1e3, log=True)\n" + + " return (x - 2) ** 2 + y + lx\n" + + "study.optimize(_obj, n_trials=24)\n"); + } + + /// QMC reads its first finished trial's distributions from a compact record and matches Python bit for bit. + [TestMethod] + [TestCategory("RequiresPython")] + public void Qmc_FullState_IsBitIdentical() + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy(storage: CompactStorage(), studyName: "compact", + sampler: new QMCSampler(seed: 3)); + csStudy.Optimize(t => t.SuggestFloat("x", -1, 1) + t.SuggestInt("n", 0, 9), nTrials: 16); + + AssertFullParity(csStudy, + "import warnings; warnings.simplefilter('ignore')\n" + + "study = optuna.create_study(study_name='compact', sampler=optuna.samplers.QMCSampler(seed=3))\n" + + "study.optimize(lambda t: t.suggest_float('x', -1, 1) + t.suggest_int('n', 0, 9), n_trials=16)\n"); + } + + /// + /// CMA-ES restores its optimizer from the hexadecimal chunks the compact storage holds as bytes: every trial + /// matches Python bit for bit, and the stored chunks are exactly the strings an uncompacted C# run stores + /// (whose pickles already proves equal to Python's). + /// + /// Sampler seed. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(0)] + [DataRow(7)] + public void CmaEs_FullState_IsBitIdentical_AndStoresTheSameChunks(int seed) + { + PythonSession.EnsureBitExactHostOrInconclusive(); + + // Summed left to right on both sides: Python's sum() compensates (Neumaier, CPython 3.12+), which would change + // the last bit of some values for a reason that has nothing to do with the storage. + Func sphere = t => + { + double v = 0; + for (int i = 0; i < 3; i++) + v += Math.Pow(t.SuggestFloat($"x{i}", -3, 3) - 1, 2); + return v; + }; + var csStudy = Optuna.CreateStudy(storage: CompactStorage(), studyName: "cma", sampler: new CmaEsSampler(seed: seed)); + csStudy.Optimize(sphere, nTrials: 28); + var plainStudy = Optuna.CreateStudy(storage: new InMemoryStorage(), studyName: "cma", sampler: new CmaEsSampler(seed: seed)); + plainStudy.Optimize(sphere, nTrials: 28); + AssertRecordsStayedCompact(csStudy); + + var csState = StudySnapshot.Capture(csStudy); + var plainState = StudySnapshot.Capture(plainStudy); + int chunks = 0; + for (int i = 0; i < csState.Trials.Count; i++) + { + var compactChunks = csState.Trials[i].SystemAttrs.Where(kv => kv.Key.Contains(":optimizer:")).ToList(); + var plainChunks = plainState.Trials[i].SystemAttrs.Where(kv => kv.Key.Contains(":optimizer:")).ToList(); + compactChunks.Select(kv => kv.Key).Should().Equal(plainChunks.Select(kv => kv.Key)); + compactChunks.Select(kv => (string)kv.Value!).Should().Equal(plainChunks.Select(kv => (string)kv.Value!)); + chunks += compactChunks.Count; + } + chunks.Should().BePositive("the study must have stored optimizer states"); + + PyExec( + $"study = optuna.create_study(study_name='cma', sampler=optuna.samplers.CmaEsSampler(seed={seed}))\n" + + "def _obj(t):\n" + + " v = 0.0\n" + + " for i in range(3):\n" + + " v += (t.suggest_float(f'x{i}', -3, 3) - 1) ** 2\n" + + " return v\n" + + "study.optimize(_obj, n_trials=28)\n"); + var pyState = PyStudyState("study"); + foreach (var snapshot in new[] { pyState, csState }) + foreach (var trial in snapshot.Trials) + foreach (string key in trial.SystemAttrs.Keys.Where(k => k.Contains(":optimizer:")).ToList()) + trial.SystemAttrs.Remove(key); + StudySnapshot.AssertParity(pyState, csState, DatetimeComparison.NullnessOnly); + } + + /// + /// Multivariate TPE reads every finished trial's parameters on every ask through the compact views and stays in + /// lock-step with Python: same trial count, states and parameter sets, continuous values within TPE's documented + /// erf tolerance and categorical values exact. + /// + /// Sampler seed. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(0)] + [DataRow(42)] + public void TpeMultivariate_TrajectoryMatches(int seed) + { + PythonSession.EnsureOrInconclusive(); + const double tolerance = 1e-9; + + var csStudy = Optuna.CreateStudy(storage: CompactStorage(), studyName: "tpe", sampler: new TPESampler(seed: seed)); + csStudy.Optimize(t => + { + double x1 = t.SuggestFloat("x1", -5, 5); + t.SuggestFloat("x2", -5, 5); + t.SuggestFloat("lx", 1e-3, 1e3, log: true); + t.SuggestCategorical("cc", new CategoricalChoice[] { "a", "b", "c" }); + return x1 * x1; + }, nTrials: 30); + AssertRecordsStayedCompact(csStudy); + var csState = StudySnapshot.Capture(csStudy); + + PyExec( + $"study = optuna.create_study(study_name='tpe', sampler=optuna.samplers.TPESampler(seed={seed}))\n" + + "def _obj(t):\n" + + " x1 = t.suggest_float('x1', -5, 5)\n" + + " t.suggest_float('x2', -5, 5)\n" + + " t.suggest_float('lx', 1e-3, 1e3, log=True)\n" + + " t.suggest_categorical('cc', ['a', 'b', 'c'])\n" + + " return x1 ** 2\n" + + "study.optimize(_obj, n_trials=30)\n"); + var pyState = PyStudyState("study"); + + csState.Trials.Count.Should().Be(pyState.Trials.Count); + for (int i = 0; i < pyState.Trials.Count; i++) + { + var expected = pyState.Trials[i]; + var actual = csState.Trials[i]; + actual.State.Should().Be(expected.State, $"trial {i} state"); + actual.Params.Keys.Should().BeEquivalentTo(expected.Params.Keys, $"trial {i} parameter set"); + foreach (string key in expected.Params.Keys) + { + if (expected.Params[key] is double e && actual.Params[key] is double a) + a.Should().BeApproximately(e, Math.Max(tolerance, tolerance * Math.Abs(e)), $"trial {i} '{key}'"); + else + actual.Params[key].Should().Be(expected.Params[key], $"trial {i} '{key}'"); + } + } + } +} diff --git a/test/OptunaSharp.Tests/Storages/CompactFinishedTrialsTests.cs b/test/OptunaSharp.Tests/Storages/CompactFinishedTrialsTests.cs new file mode 100644 index 0000000..60e1292 --- /dev/null +++ b/test/OptunaSharp.Tests/Storages/CompactFinishedTrialsTests.cs @@ -0,0 +1,973 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using AwesomeAssertions; +using OptunaSharp.Collections; +using OptunaSharp.Distributions; +using OptunaSharp.Internal; +using OptunaSharp.Logging; +using OptunaSharp.Pruners; +using OptunaSharp.Samplers; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Storages; + +/// +/// Plan 13 (docs/plans/13-rustuna-derived-performance.md), item S12: InMemoryStorage(compactFinishedTrials: true) +/// stores each finished trial compactly (parameters as internal values against a schema the study's trials share, +/// attributes as arrays with long ASCII strings as bytes, nothing for empty dictionaries) while every API returns the +/// same contents. The tests compare a compact storage against Optuna's representation value by value (types and bits), +/// prove the read-only views refuse writes and the mutable accessors keep writes, pin the fallbacks that keep a +/// dictionary when a value would not read back identical, and run every sampler family twice (compact and not) to the +/// same seeded trials, with the records still compact afterwards. +/// +[TestClass] +public class CompactFinishedTrialsTests +{ + private static readonly StudyDirection[] Minimize = { StudyDirection.Minimize }; + + // ------------------------------------------------------------------ fixtures + + /// + /// A study's worth of trials written through the storage API with every kind of value: each float, integer and + /// categorical flavor (a -0.0 value, log and stepped ranges, choices of every JSON type), attributes with short, + /// long ASCII, long odd-length lowercase hexadecimal, uppercase hexadecimal and non-ASCII strings, nested + /// containers, intermediate values (and a trial without), a failed and a pruned trial. + /// + /// Whether the storage compacts finished trials. + /// The storage and the study it holds. + private static (InMemoryStorage Storage, int StudyId) WriteEveryKind(bool compact) + { + var storage = new InMemoryStorage(compactFinishedTrials: compact); + int studyId = storage.CreateNewStudy(Minimize, "every-kind"); + var categorical = new CategoricalDistribution( + new CategoricalChoice[] { "a", CategoricalChoice.None, true, 7L, 2.5, "z" }); + for (int k = 0; k < 4; k++) + { + int trialId = storage.CreateNewTrial(studyId); + storage.SetTrialParam(trialId, "x", 0.25 * k - 0.5, new FloatDistribution(-1, 1)); + storage.SetTrialParam(trialId, "neg0", -0.0, new FloatDistribution(-1, 1)); + storage.SetTrialParam(trialId, "lx", 0.01 * (k + 1), new FloatDistribution(1e-3, 1, log: true)); + storage.SetTrialParam(trialId, "sx", 0.25 * k, new FloatDistribution(0, 1, step: 0.25)); + storage.SetTrialParam(trialId, "n", k - 2, new IntDistribution(-3, 3)); + storage.SetTrialParam(trialId, "ln", 1 << k, new IntDistribution(1, 64, log: true)); + storage.SetTrialParam(trialId, "sn", 2 * k, new IntDistribution(0, 10, step: 2)); + // A new categorical distribution per trial, as every SuggestCategorical builds one. + var perTrialCategorical = new CategoricalDistribution( + new CategoricalChoice[] { "a", CategoricalChoice.None, true, 7L, 2.5, "z" }); + storage.SetTrialParam(trialId, "c", k, k % 2 == 0 ? categorical : perTrialCategorical); + storage.SetTrialParam(trialId, "c5", 5, perTrialCategorical); + + storage.SetTrialUserAttr(trialId, "short", "s"); + storage.SetTrialUserAttr(trialId, "ascii", new string('q', 100) + k); + storage.SetTrialUserAttr(trialId, "hex", LowerHex(65 + k)); + storage.SetTrialUserAttr(trialId, "upper-hex", LowerHex(80).ToUpperInvariant()); + storage.SetTrialUserAttr(trialId, "unicode", new string('é', 70)); + storage.SetTrialUserAttr(trialId, "nested", new List { 1L, "two", 3.5, null, true }); + storage.SetTrialUserAttr(trialId, "map", new OrderedDict { ["a"] = 1L, ["b"] = -0.0 }); + storage.SetTrialUserAttr(trialId, "none", null); + storage.SetTrialSystemAttr(trialId, "gen", (long)k); + storage.SetTrialSystemAttr(trialId, "chunk:0", LowerHex(2045)); + storage.SetTrialSystemAttr(trialId, "constraints", new List { 0.5 - k, -1.0 }); + if (k != 1) + { + storage.SetTrialIntermediateValue(trialId, 0, 1.5 * k); + storage.SetTrialIntermediateValue(trialId, 7, -0.0); + storage.SetTrialIntermediateValue(trialId, long.MaxValue, double.NegativeInfinity); + } + TrialState state = k switch { 2 => TrialState.Fail, 3 => TrialState.Pruned, _ => TrialState.Complete }; + storage.SetTrialStateValues(trialId, state, state == TrialState.Complete ? new[] { 1.0 / (k + 1) } : null); + } + return (storage, studyId); + } + + /// A lowercase hexadecimal string of the given length (odd lengths included). + /// Number of digits. + /// Digits cycling through 0-9a-f. + private static string LowerHex(int length) + { + var sb = new StringBuilder(length); + for (int i = 0; i < length; i++) + sb.Append("0123456789abcdef"[(i * 7 + 3) % 16]); + return sb.ToString(); + } + + /// + /// Everything a reader can learn from a trial except object identity and the clock: values with their runtime + /// types and bits, entries in order, the distributions' reprs and the trial's repr without its datetimes. + /// + /// The trial to describe; read through the read-only accessors only. + /// A string equal for two trials exactly when no read can tell them apart. + private static string Fingerprint(FrozenTrial trial) + { + var sb = new StringBuilder(); + sb.Append(trial.Number).Append('|').Append(trial.State).Append("|values="); + sb.Append(trial.Values is null + ? "None" + : string.Join(",", trial.Values.Select(v => BitConverter.DoubleToInt64Bits(v)))); + foreach (var (name, value) in trial.Params) + sb.Append("|p:").Append(name).Append('=').Append(Describe(value)); + foreach (var (name, distribution) in trial.Distributions) + sb.Append("|d:").Append(name).Append('=').Append(distribution.GetType().Name).Append(distribution); + foreach (var (key, value) in trial.UserAttrs) + sb.Append("|u:").Append(key).Append('=').Append(Describe(value)); + foreach (var (key, value) in trial.SystemAttrs) + sb.Append("|s:").Append(key).Append('=').Append(Describe(value)); + foreach (var (step, value) in trial.IntermediateValuesForRead) + sb.Append("|i:").Append(step).Append('=').Append(BitConverter.DoubleToInt64Bits(value)); + sb.Append("|last=").Append(trial.LastStep); + sb.Append("|constraints=").Append(string.Join(",", trial.Constraints.Select(kv => $"{kv.Key}:{BitConverter.DoubleToInt64Bits(kv.Value)}"))); + sb.Append("|repr=").Append(WithoutDatetimes(trial.ToString())); + return sb.ToString(); + } + + /// A value's runtime type and Python repr, with doubles by their bits so -0.0 and 0.0 differ. + /// The value to describe. + /// Type name and representation. + private static string Describe(object? value) => value switch + { + null => "None", + double d => $"double:{BitConverter.DoubleToInt64Bits(d)}", + _ => $"{value.GetType().Name}:{PythonRepr.Repr(value)}", + }; + + /// A trial repr with its two datetimes blanked, the one thing two storages cannot agree on. + /// The repr from . + /// The repr with datetime_start and datetime_complete values removed. + private static string WithoutDatetimes(string repr) + => Regex.Replace(repr, @"datetime_(start|complete)=('[^']*'|None)", "datetime_$1=?"); + + // ------------------------------------------------------------------ the switch and what it touches + + /// The default storage keeps Optuna's representation: no record is ever compact, every dictionary an OrderedDict. + [TestMethod] + public void Default_IsOff_AndKeepsEveryDictionary() + { + new InMemoryStorage().CompactFinishedTrials.Should().BeFalse(); + new InMemoryStorage(compactFinishedTrials: false).CompactFinishedTrials.Should().BeFalse(); + new InMemoryStorage(compactFinishedTrials: true).CompactFinishedTrials.Should().BeTrue(); + + var (storage, studyId) = WriteEveryKind(compact: false); + foreach (FrozenTrial trial in storage.GetAllTrials(studyId, deepcopy: false)) + { + trial.IsCompact.Should().BeFalse(); + trial.Params.Should().BeOfType>(); + trial.Distributions.Should().BeOfType>(); + trial.UserAttrs.Should().BeOfType>(); + trial.SystemAttrs.Should().BeOfType>(); + } + } + + /// + /// Only finished records are compacted: a running and a waiting trial keep their dictionaries (they are still + /// written), each finished state (complete, failed, pruned) is compact, and a trial added already finished is + /// compacted at creation. + /// + [TestMethod] + public void OnlyFinishedRecords_AreCompacted() + { + var storage = new InMemoryStorage(compactFinishedTrials: true); + var study = Optuna.CreateStudy(storage: storage, sampler: new RandomSampler(seed: 1)); + study.EnqueueTrial(new OrderedDict { ["x"] = 0.5 }); + Trial running = study.Ask(); // pops the enqueued trial + running.SuggestFloat("x", 0, 1); + study.EnqueueTrial(new OrderedDict { ["x"] = 0.25 }); + study.AddTrial(FrozenTrial.CreateTrial( + value: 3.0, + @params: new OrderedDict { ["x"] = 0.75 }, + distributions: new OrderedDict { ["x"] = new FloatDistribution(0, 1) }, + userAttrs: new OrderedDict { ["added"] = true })); + study.Optimize(t => t.SuggestFloat("x", 0, 1), nTrials: 3); + + var records = storage.GetAllTrials(study.StudyId, deepcopy: false); + foreach (FrozenTrial record in records) + record.IsCompact.Should().Be(record.State.IsFinished(), $"trial {record.Number} is {record.State}"); + records.Count(r => r.State == TrialState.Running).Should().Be(1); + records.Single(r => r.UserAttrs.ContainsKey("added")).Params["x"].Should().Be(0.75); + } + + // ------------------------------------------------------------------ contents: identical to Optuna's representation + + /// + /// Every kind of value reads back from a compact record exactly as from Optuna's representation: same runtime + /// types, same bits (-0.0 stays -0.0), same order, same reprs and constraints, the same LastStep; and the + /// compact storage did compact every finished record, with the long strings stored as bytes (hexadecimal two to a + /// byte, odd length included) and the uppercase-hex and non-ASCII strings left as strings. + /// + [TestMethod] + public void EveryKindOfValue_ReadsBackIdentical() + { + var (plain, plainStudy) = WriteEveryKind(compact: false); + var (compact, compactStudy) = WriteEveryKind(compact: true); + var expected = plain.GetAllTrials(plainStudy, deepcopy: false); + var actual = compact.GetAllTrials(compactStudy, deepcopy: false); + + actual.Select(Fingerprint).Should().Equal(expected.Select(Fingerprint)); + actual.Should().OnlyContain(t => t.IsCompact); + actual.Select(t => t.Params).Should().AllBeOfType(); + + var userAttrs = (CompactAttrs)actual[0].UserAttrs; + userAttrs.StoredValue("ascii").Should().BeOfType().Which.IsHex.Should().BeFalse(); + var hex = userAttrs.StoredValue("hex").Should().BeOfType().Which; + hex.IsHex.Should().BeTrue(); + hex.StoredBytes.Should().Be(32, "65 digits are 32 bytes and one spare digit"); + userAttrs.StoredValue("upper-hex").Should().BeOfType().Which.IsHex.Should().BeFalse(); + userAttrs.StoredValue("unicode").Should().BeOfType(); + userAttrs.StoredValue("short").Should().BeOfType(); + ((CompactAttrs)actual[0].SystemAttrs).StoredValue("chunk:0").Should().BeOfType() + .Which.StoredBytes.Should().Be(1022, "a 2,045-digit CMA-ES chunk is 1,022 bytes and one spare digit"); + actual[1].IntermediateValuesForRead.Should().BeEmpty(); + actual[1].LastStep.Should().BeNull(); + + // The storage's own internal-value read returns the stored value of a compact trial without converting it. + foreach (FrozenTrial trial in expected) + { + foreach (string name in trial.Params.Keys) + BitConverter.DoubleToInt64Bits(compact.GetTrialParam(trial.TrialId, name)) + .Should().Be(BitConverter.DoubleToInt64Bits(plain.GetTrialParam(trial.TrialId, name)), name); + Assert.ThrowsExactly(() => plain.GetTrialParam(trial.TrialId, "missing")); + Assert.ThrowsExactly(() => compact.GetTrialParam(trial.TrialId, "missing")); + } + } + + /// + /// A compact record equals the same trial in Optuna's representation, both ways, and prints the same repr: the + /// equality and repr paths read the views, not the dictionaries. + /// + [TestMethod] + public void Equals_AndRepr_MatchOptunasRepresentation() + { + var (plain, plainStudy) = WriteEveryKind(compact: false); + var (compact, compactStudy) = WriteEveryKind(compact: true); + var expected = plain.GetAllTrials(plainStudy, deepcopy: false); + var actual = compact.GetAllTrials(compactStudy, deepcopy: false); + for (int i = 0; i < actual.Count; i++) + { + FrozenTrial twin = expected[i].DeepCopy(); + twin.SetDatetimeStart(actual[i].DatetimeStart); + twin.DatetimeComplete = actual[i].DatetimeComplete; + twin.TrialId = actual[i].TrialId; + + actual[i].Equals(twin).Should().BeTrue($"trial {i} compact equals the dictionary form"); + twin.Equals(actual[i]).Should().BeTrue($"trial {i} dictionary form equals the compact one"); + actual[i].ToString().Should().Be(twin.ToString()); + actual[i].IsCompact.Should().BeTrue("comparing and printing read the views only"); + } + } + + /// + /// A deep copy of a compact record holds real dictionaries with the same contents and leaves the record compact; + /// a shallow copy first gives the record dictionaries of its own and shares them, as Python's copy.copy + /// shares a trial's dicts. + /// + [TestMethod] + public void Copies_DeepHoldDictionaries_ShallowShareTheRecordsOwn() + { + var (compact, studyId) = WriteEveryKind(compact: true); + FrozenTrial record = compact.GetAllTrials(studyId, deepcopy: false)[0]; + + FrozenTrial deep = record.DeepCopy(); + deep.IsCompact.Should().BeFalse(); + deep.Params.Should().BeOfType>(); + Fingerprint(deep).Should().Be(Fingerprint(record)); + record.IsCompact.Should().BeTrue(); + compact.GetAllTrials(studyId, deepcopy: true).Select(Fingerprint) + .Should().Equal(compact.GetAllTrials(studyId, deepcopy: false).Select(Fingerprint)); + + string before = Fingerprint(record); + FrozenTrial shallow = record.ShallowCopy(); + record.IsCompact.Should().BeFalse("a shallow copy shares real dictionaries"); + shallow.ParamsDict.Should().BeSameAs(record.ParamsDict); + shallow.DistributionsDict.Should().BeSameAs(record.DistributionsDict); + shallow.UserAttrsDict.Should().BeSameAs(record.UserAttrsDict); + shallow.SystemAttrsDict.Should().BeSameAs(record.SystemAttrsDict); + shallow.IntermediateValues.Should().BeSameAs(record.IntermediateValues); + Fingerprint(record).Should().Be(before); + shallow.ParamsDict["x"] = 0.125; + record.Params["x"].Should().Be(0.125, "the copy and the record share the dictionary"); + } + + // ------------------------------------------------------------------ the views and the mutable accessors + + /// + /// The read-only views behave as dictionaries for reads (indexer, lookups, ordered keys and values, the + /// non-generic interface PythonRepr and PyJson use, CopyTo) and refuse every write through + /// ; a missing key throws the same exception, with the same message, as an + /// . + /// + [TestMethod] + public void Views_ReadLikeDictionaries_AndRefuseWrites() + { + var (plain, plainStudy) = WriteEveryKind(compact: false); + var (compact, compactStudy) = WriteEveryKind(compact: true); + FrozenTrial expected = plain.GetAllTrials(plainStudy, deepcopy: false)[0]; + FrozenTrial actual = compact.GetAllTrials(compactStudy, deepcopy: false)[0]; + + void CheckView(IReadOnlyDictionary view, IReadOnlyDictionary reference) + { + view.Should().NotBeAssignableTo>(); + view.Count.Should().Be(reference.Count); + view.Keys.Should().Equal(reference.Keys); + view.Values.Select(v => Describe(v)).Should().Equal(reference.Values.Select(v => Describe(v))); + foreach (string key in reference.Keys) + { + view.ContainsKey(key).Should().BeTrue(); + view.TryGetValue(key, out TValue? value).Should().BeTrue(); + Describe(value).Should().Be(Describe(reference[key])); + Describe(view[key]).Should().Be(Describe(reference[key])); + } + view.ContainsKey("missing").Should().BeFalse(); + view.TryGetValue("missing", out _).Should().BeFalse(); + string expectedMessage = Assert.ThrowsExactly(() => _ = reference["missing"]).Message; + Assert.ThrowsExactly(() => _ = view["missing"]).Message.Should().Be(expectedMessage); + Assert.ThrowsExactly(() => view.ContainsKey(null!)); + + var nonGeneric = (IDictionary)view; + var referenceNonGeneric = (IDictionary)reference; + nonGeneric.IsReadOnly.Should().BeTrue(); + nonGeneric.IsFixedSize.Should().BeTrue(); + nonGeneric.Count.Should().Be(reference.Count); + nonGeneric.Keys.Cast().Should().Equal(referenceNonGeneric.Keys.Cast()); + nonGeneric.Values.Cast().Select(Describe).Should().Equal(referenceNonGeneric.Values.Cast().Select(Describe)); + nonGeneric.Contains(reference.Keys.First()).Should().BeTrue(); + nonGeneric.Contains(42).Should().BeFalse("a key of another type is absent, as in a dictionary"); + nonGeneric["missing"].Should().BeNull(); + nonGeneric[42].Should().BeNull(); + var entries = new List(); + IDictionaryEnumerator enumerator = nonGeneric.GetEnumerator(); + while (enumerator.MoveNext()) + entries.Add($"{enumerator.Key}={Describe(enumerator.Value)}={Describe(enumerator.Entry.Value)}"); + entries.Should().Equal(reference.Select(kv => $"{kv.Key}={Describe(kv.Value)}={Describe(kv.Value)}")); + enumerator.MoveNext().Should().BeFalse(); + enumerator.Reset(); + enumerator.MoveNext().Should().BeTrue(); + enumerator.Key.Should().Be(reference.Keys.First()); + + var pairs = new KeyValuePair[reference.Count + 1]; + nonGeneric.CopyTo(pairs, 1); + pairs.Skip(1).Select(kv => kv.Key).Should().Equal(reference.Keys); + var dictionaryEntries = new DictionaryEntry[reference.Count]; + nonGeneric.CopyTo(dictionaryEntries, 0); + dictionaryEntries.Select(e => (string)e.Key).Should().Equal(reference.Keys); + var objects = new object?[reference.Count]; + nonGeneric.CopyTo(objects, 0); + objects.Should().AllBeOfType>(); + Assert.ThrowsExactly(() => nonGeneric.CopyTo(new int[reference.Count], 0)); + Assert.ThrowsExactly(() => nonGeneric.CopyTo(new object?[reference.Count - 1], 0)); + Assert.ThrowsExactly(() => nonGeneric.CopyTo(objects, -1)); + nonGeneric.IsSynchronized.Should().BeFalse(); + nonGeneric.SyncRoot.Should().BeSameAs(view); + + Assert.ThrowsExactly(() => nonGeneric["k"] = 1); + Assert.ThrowsExactly(() => nonGeneric.Add("k", 1)); + Assert.ThrowsExactly(() => nonGeneric.Remove(reference.Keys.First())); + Assert.ThrowsExactly(() => nonGeneric.Clear()); + + PythonRepr.Repr(view).Should().Be(PythonRepr.Repr(reference)); + } + + CheckView(actual.Params, expected.Params); + CheckView(actual.Distributions, expected.Distributions); + CheckView(actual.UserAttrs, expected.UserAttrs); + CheckView(actual.SystemAttrs, expected.SystemAttrs); + PyJson.Dumps(actual.UserAttrs).Should().Be(PyJson.Dumps(expected.UserAttrs)); + PyJson.Dumps(actual.Params).Should().Be(PyJson.Dumps(expected.Params)); + actual.IsCompact.Should().BeTrue("every read above went through the views"); + } + + /// + /// Each mutable accessor gives a compact record a real dictionary of its own, returns that same dictionary on every + /// later read, and keeps a change made through it; the other dictionaries stay compact until their own accessor + /// is read. and write through them. + /// + [TestMethod] + public void MutableAccessors_GiveTheRecordItsOwnDictionary_AndKeepWrites() + { + var (compact, studyId) = WriteEveryKind(compact: true); + var records = compact.GetAllTrials(studyId, deepcopy: false); + FrozenTrial trial = records[0]; + string before = Fingerprint(trial); + + OrderedDict @params = trial.ParamsDict; + Fingerprint(trial).Should().Be(before, "turning a dictionary back changes no content"); + trial.ParamsDict.Should().BeSameAs(@params); + trial.Params.Should().BeSameAs(@params); + trial.Distributions.Should().BeOfType("only the parameters were asked for"); + @params["x"] = 0.875; + trial.Params["x"].Should().Be(0.875); + + OrderedDict distributions = trial.DistributionsDict; + trial.DistributionsDict.Should().BeSameAs(distributions); + distributions["y"] = new FloatDistribution(0, 2); + trial.Distributions["y"].Should().Be(new FloatDistribution(0, 2)); + + OrderedDict userAttrs = trial.UserAttrsDict; + trial.UserAttrsDict.Should().BeSameAs(userAttrs); + userAttrs["ascii"].Should().Be(new string('q', 100) + 0, "a compacted string comes back as the string"); + trial.SetUserAttr("added", 1L); + trial.UserAttrs["added"].Should().Be(1L); + + OrderedDict systemAttrs = trial.SystemAttrsDict; + trial.SystemAttrsDict.Should().BeSameAs(systemAttrs); + trial.SetConstraint("c", -2.0); + trial.SystemAttrs.Should().ContainKey("constraints:c"); + + FrozenTrial noReports = records[1]; + noReports.IntermediateValuesForRead.Should().BeEmpty(); + noReports.IsCompact.Should().BeTrue(); + OrderedDict intermediate = noReports.IntermediateValues; + noReports.IntermediateValues.Should().BeSameAs(intermediate); + intermediate[3] = 1.0; + noReports.IntermediateValues[3].Should().Be(1.0); + noReports.LastStep.Should().Be(3); + + trial.IsCompact.Should().BeFalse(); + records[2].IsCompact.Should().BeTrue("no other record was touched"); + } + + /// + /// Many threads asking the same compact record for a mutable dictionary at once all get the one dictionary the + /// record keeps (the compare-exchange in FrozenTrial.Decompact), for every accessor, over many records. + /// + [TestMethod] + public void MutableAccessors_UnderContention_AllReturnTheOneDictionary() + { + var storage = new InMemoryStorage(compactFinishedTrials: true); + var study = Optuna.CreateStudy(storage: storage, sampler: new RandomSampler(seed: 3)); + study.Optimize(t => + { + t.SetUserAttr("u", 1L); + return t.SuggestFloat("x", 0, 1) + t.SuggestInt("n", 0, 3); + }, nTrials: 64); + var records = storage.GetAllTrials(study.StudyId, deepcopy: false); + records.Should().OnlyContain(r => r.IsCompact); + + const int threads = 8; + foreach (FrozenTrial record in records) + { + var seen = new object[threads][]; + using var start = new Barrier(threads); + Parallel.For(0, threads, new ParallelOptions { MaxDegreeOfParallelism = threads }, i => + { + start.SignalAndWait(); + seen[i] = new object[] + { + record.ParamsDict, record.DistributionsDict, record.UserAttrsDict, record.SystemAttrsDict, + record.IntermediateValues, + }; + }); + for (int i = 1; i < threads; i++) + for (int field = 0; field < seen[0].Length; field++) + seen[i][field].Should().BeSameAs(seen[0][field], $"trial {record.Number}, field {field}, thread {i}"); + ((object)record.ParamsDict).Should().BeSameAs(seen[0][0]); + ((object)record.IntermediateValues).Should().BeSameAs(seen[0][4]); + } + } + + // ------------------------------------------------------------------ fallbacks: keep the dictionary when not exact + + /// + /// A trial whose parameters would not read back identical keeps its parameter dictionary (while its attributes are + /// still compacted): an int under a float distribution (Optuna keeps it an int), an integer beyond + /// 2^53, a categorical choice Python-equal to an earlier one (1.0 after 1), a value its distribution + /// cannot convert, and names listed in another order than the distributions. + /// + /// Which inexact trial to add. + [TestMethod] + [DataRow("int-under-float")] + [DataRow("beyond-2^53")] + [DataRow("python-equal-choice")] + [DataRow("unconvertible")] + [DataRow("order")] + public void InexactParameters_KeepTheirDictionary(string @case) + { + var storage = new InMemoryStorage(compactFinishedTrials: true); + int studyId = storage.CreateNewStudy(Minimize, "inexact"); + OrderedDict @params; + OrderedDict distributions; + switch (@case) + { + case "int-under-float": + @params = new() { ["x"] = 1 }; + distributions = new() { ["x"] = new FloatDistribution(0, 2) }; + break; + case "beyond-2^53": + @params = new() { ["n"] = (1L << 53) + 1 }; + distributions = new() { ["n"] = new IntDistribution(0, long.MaxValue / 2) }; + break; + case "python-equal-choice": + @params = new() { ["c"] = 1.0 }; + distributions = new() { ["c"] = new CategoricalDistribution(new CategoricalChoice[] { 1L, 1.0 }) }; + break; + case "unconvertible": + @params = new() { ["x"] = "not a number" }; + distributions = new() { ["x"] = new FloatDistribution(0, 2) }; + break; + default: + @params = new() { ["x"] = 0.5, ["y"] = 0.25 }; + distributions = new() { ["y"] = new FloatDistribution(0, 1), ["x"] = new FloatDistribution(0, 1) }; + break; + } + // Built directly (CreateTrial would reject the unconvertible value); the storage adds it already finished. + var template = new FrozenTrial( + number: -1, state: TrialState.Complete, value: 1.0, datetimeStart: DateTime.Now, + datetimeComplete: DateTime.Now, @params: @params, distributions: distributions, + userAttrs: new OrderedDict { ["u"] = new string('a', 80) }, + systemAttrs: new OrderedDict(), intermediateValues: new OrderedDict(), + trialId: -1); + int trialId = storage.CreateNewTrial(studyId, template); + + FrozenTrial record = storage.GetTrial(trialId); + record.Params.Should().BeOfType>(); + record.Distributions.Should().BeOfType>(); + record.UserAttrs.Should().BeOfType("the attributes compact independently"); + record.IsCompact.Should().BeTrue(); + foreach (var (name, value) in @params) + Describe(record.Params[name]).Should().Be(Describe(value), "the stored value, type and all"); + record.Params.Keys.Should().Equal(@params.Keys); + } + + /// + /// Trials of one search space share one schema, so one distributions view, even with a new categorical + /// distribution per trial; a dynamic space keeps one schema per shape; past + /// shapes, new shapes keep their dictionaries. + /// + [TestMethod] + public void Schemas_AreSharedPerShape_AndBounded() + { + var storage = new InMemoryStorage(compactFinishedTrials: true); + var study = Optuna.CreateStudy(storage: storage, sampler: new RandomSampler(seed: 5)); + study.Optimize(t => + { + double x = t.SuggestFloat("x", 0, 1); + CategoricalChoice c = t.SuggestCategorical("c", new CategoricalChoice[] { "a", "b", 3L }); + // A conditional parameter: two shapes. + if (c.Equals((CategoricalChoice)"a")) + t.SuggestInt("only-a", 0, 5); + return x; + }, nTrials: 40); + var records = storage.GetAllTrials(study.StudyId, deepcopy: false); + records.Should().OnlyContain(r => r.Params is CompactParams); + var views = records.Select(r => r.Distributions).Distinct(ReferenceEqualityComparer.Instance).ToList(); + views.Should().HaveCount(2, "one schema per shape, shared across every trial of that shape"); + records.GroupBy(r => r.Params.ContainsKey("only-a")) + .Should().OnlyContain(g => g.Select(r => r.Distributions).Distinct(ReferenceEqualityComparer.Instance).Count() == 1); + + var cache = new ParamSchemaCache(); + var distribution = new FloatDistribution(0, 1); + for (int i = 0; i < ParamSchemaCache.MaxSchemas + 10; i++) + { + ParamSchema? schema = cache.FindOrAdd( + new OrderedDict { [$"p{i}"] = 0.5 }, + new OrderedDict { [$"p{i}"] = distribution }); + if (i < ParamSchemaCache.MaxSchemas) + schema.Should().NotBeNull(); + else + schema.Should().BeNull("the cache is full"); + } + cache.Count.Should().Be(ParamSchemaCache.MaxSchemas); + cache.FindOrAdd(new OrderedDict { ["p3"] = 0.1 }, + new OrderedDict { ["p3"] = new FloatDistribution(0, 1) }) + .Should().NotBeNull("a known shape is still found when the cache is full"); + } + + /// + /// The value and distribution identity rules: bits for floating point (-0.0 is not 0.0), type for numbers (1 is + /// not 1.0), characters for strings, the same instance for anything else; categoricals by identical choices, + /// float and integer distributions bitwise, a categorical subclass by reference only; hashes agree with matches. + /// + [TestMethod] + public void IdentityRules_MatchWhatAReaderCanSee() + { + DistributionIdentity.IsSameValue(0.0, -0.0).Should().BeFalse(); + DistributionIdentity.IsSameValue(1.5, 1.5).Should().BeTrue(); + DistributionIdentity.IsSameValue(1L, 1.0).Should().BeFalse(); + DistributionIdentity.IsSameValue(1L, 1).Should().BeFalse("long and int are different types"); + DistributionIdentity.IsSameValue(2f, 2f).Should().BeTrue(); + DistributionIdentity.IsSameValue(true, true).Should().BeTrue(); + DistributionIdentity.IsSameValue("ab", new string(new[] { 'a', 'b' })).Should().BeTrue(); + DistributionIdentity.IsSameValue(null, null).Should().BeTrue(); + DistributionIdentity.IsSameValue(null, "x").Should().BeFalse(); + DistributionIdentity.IsSameValue(1.0m, 1.00m).Should().BeFalse("decimals print their scale"); + DistributionIdentity.IsSameValue(new List(), new List()).Should().BeFalse(); + + CategoricalDistribution Categorical(params CategoricalChoice[] choices) => new(choices); + var pairs = new (BaseDistribution A, BaseDistribution B, bool Interchangeable)[] + { + (Categorical("a", 2L, 2.5), Categorical("a", 2L, 2.5), true), + (Categorical("a", CategoricalChoice.None), Categorical("a", CategoricalChoice.None), true), + (Categorical(1L), Categorical(1.0), false), + (Categorical("a"), Categorical("a", "b"), false), + (new FloatDistribution(0, 1), new FloatDistribution(0, 1), true), + (new FloatDistribution(-0.0, 1), new FloatDistribution(0, 1), false), + (new IntDistribution(0, 9, step: 3), new IntDistribution(0, 9, step: 3), true), + (new IntDistribution(0, 9), new FloatDistribution(0, 9), false), + (new EnumLikeCategorical("a"), new EnumLikeCategorical("a"), false), + }; + foreach (var (a, b, interchangeable) in pairs) + { + DistributionIdentity.AreInterchangeable(a, b).Should().Be(interchangeable, $"{a} vs {b}"); + DistributionIdentity.AreInterchangeable(b, a).Should().Be(interchangeable, $"{b} vs {a}"); + if (interchangeable) + DistributionIdentity.HashOf(a).Should().Be(DistributionIdentity.HashOf(b)); + } + var same = new EnumLikeCategorical("a"); + DistributionIdentity.AreInterchangeable(same, same).Should().BeTrue(); + } + + /// + /// The fast path's positions: a reader's distribution that converts as the schema's does gets the stored + /// internal value's position, a missing name , and a Python-equal but not + /// identical categorical ; at every position the stored value is + /// exactly what the reader's conversion of the external value returns. + /// + [TestMethod] + public void SchemaPositions_ReturnTheStoredValueOnlyWhereItIsExact() + { + var x = new FloatDistribution(0, 1); + var c = new CategoricalDistribution(new CategoricalChoice[] { 1L, 2L }); + var @params = new OrderedDict { ["x"] = 0.25, ["c"] = 2L }; + var cache = new ParamSchemaCache(); + ParamSchema schema = cache.FindOrAdd(@params, new OrderedDict { ["x"] = x, ["c"] = c })!; + CompactParams compact = CompactParams.TryCreate(schema, @params)!; + + var readers = new BaseDistribution[] + { + new CategoricalDistribution(new CategoricalChoice[] { 1L, 2L }), new FloatDistribution(0, 1), + new FloatDistribution(0, 1), new CategoricalDistribution(new CategoricalChoice[] { 1.0, 2.0 }), + }; + int[] positions = schema.PositionsFor(new[] { "c", "x", "missing", "c" }, readers); + positions.Should().Equal(1, 0, ParamSchema.Absent, ParamSchema.NotInterchangeable); + compact.InternalAt(positions[0]).Should().Be(readers[0].ToInternalRepr(compact["c"])); + compact.InternalAt(positions[1]).Should().Be(readers[1].ToInternalRepr(compact["x"])); + readers[3].ToInternalRepr(compact["c"]).Should().Be(1.0, "the slow path still converts the Python-equal choice"); + compact.TryGetInternal("c", out double stored).Should().BeTrue(); + stored.Should().Be(1.0); + compact.TryGetInternal("missing", out _).Should().BeFalse(); + } + + /// A categorical subclass: only the same instance may stand for it. + private sealed class EnumLikeCategorical : CategoricalDistribution + { + /// Builds the subclass over the given choices. + /// The choices. + public EnumLikeCategorical(params CategoricalChoice[] choices) : base(choices) + { + } + } + + /// + /// Strings: shorter than the minimum or non-ASCII stay strings; ASCII becomes one byte a character; lowercase + /// hexadecimal two digits a byte at even and odd lengths; uppercase hexadecimal is only ASCII; every stored form + /// reads back as a new string with the same characters. + /// + [TestMethod] + public void CompactStrings_RoundTripExactly() + { + CompactString.TryCreate(new string('a', CompactString.MinLength - 1)).Should().BeNull(); + CompactString.TryCreate(new string('é', 100)).Should().BeNull(); + foreach (string text in new[] + { + new string('a', CompactString.MinLength), LowerHex(64), LowerHex(65), LowerHex(2045), + LowerHex(64).ToUpperInvariant(), "0123456789" + LowerHex(70) + " tail", new string('0', 99), + }) + { + CompactString compact = CompactString.TryCreate(text)!; + compact.Should().NotBeNull(text); + string back = compact.Materialize(); + back.Should().Be(text); + back.Should().NotBeSameAs(text); + bool hex = text.All(c => c is (>= '0' and <= '9') or (>= 'a' and <= 'f')); + compact.IsHex.Should().Be(hex, text); + compact.StoredBytes.Should().Be(hex ? text.Length / 2 : text.Length); + } + } + + // ------------------------------------------------------------------ samplers: same trials, records stay compact + + /// + /// Each sampler family runs the same seeded study on Optuna's representation and on the compact storage: every + /// trial's parameters, values, attributes, intermediate values and repr are identical, and every finished record + /// is still compact at the end, so no sampler read gave a record its dictionaries back. + /// + /// The sampler (and pruner or constraint) under test. + [TestMethod] + [DataRow("random")] + [DataRow("tpe")] + [DataRow("tpe-independent")] + [DataRow("tpe-group")] + [DataRow("tpe-equal-choices")] + [DataRow("tpe-pruned")] + [DataRow("motpe-constrained")] + [DataRow("cmaes")] + [DataRow("cmaes-separable")] + [DataRow("nsgaii-constrained")] + [DataRow("nsgaiii")] + [DataRow("qmc")] + [DataRow("gp")] + [DataRow("bruteforce")] + [DataRow("grid")] + public void Samplers_GiveTheSameTrials_AndLeaveRecordsCompact(string kind) + { + List plain = RunStudy(kind, compact: false, out _); + List compact = RunStudy(kind, compact: true, out List records); + + compact.Should().Equal(plain); + records.Where(r => r.State.IsFinished()).Should().OnlyContain(r => r.IsCompact && r.Params is CompactParams, + "every sampler reads finished records through the read-only accessors"); + records.Should().Contain(r => r.State == TrialState.Complete); + } + + /// Runs one seeded study of a sampler family and describes its trials. + /// The sampler (and pruner or constraint) to use. + /// Whether the storage compacts finished trials. + /// The storage's records after the run. + /// One fingerprint per trial, in number order. + /// names no sampler family. + /// Never escapes: the pruned objective throws it inside the optimize loop, which records the trial as pruned. + private static List RunStudy(string kind, bool compact, out List records) + { + var storage = new InMemoryStorage(compactFinishedTrials: compact); + Func> constraints = t => new[] { (double)t.Params["x"]! - 1.0 }; + bool twoObjectives = kind is "motpe-constrained" or "nsgaii-constrained" or "nsgaiii"; + BaseSampler sampler = kind switch + { + "random" => new RandomSampler(seed: 11), + "tpe" => new TPESampler(seed: 11, multivariate: true, nStartupTrials: 5), + "tpe-independent" => new TPESampler(seed: 11, nStartupTrials: 5), + "tpe-group" => new TPESampler(seed: 11, multivariate: true, group: true, nStartupTrials: 5), + "tpe-equal-choices" => new TPESampler(seed: 11, multivariate: true, nStartupTrials: 5), + "tpe-pruned" => new TPESampler(seed: 11, nStartupTrials: 5), + "motpe-constrained" => new TPESampler(seed: 11, nStartupTrials: 5, constraintsFunc: constraints), + "cmaes" => new CmaEsSampler(seed: 11), + "cmaes-separable" => new CmaEsSampler(seed: 11, useSeparableCma: true), + "nsgaii-constrained" => new NSGAIISampler(populationSize: 8, seed: 11, constraintsFunc: constraints), + "nsgaiii" => new NSGAIIISampler(populationSize: 8, seed: 11), + "qmc" => new QMCSampler(seed: 11), + "gp" => new GPSampler(seed: 11, nStartupTrials: 4), + "bruteforce" => new BruteForceSampler(seed: 11), + "grid" => new GridSampler(new Dictionary> + { + ["x"] = new CategoricalChoice[] { -1.0, 0.0, 2.5 }, + ["c"] = new CategoricalChoice[] { "a", 3L }, + }, seed: 11), + _ => throw new ArgumentOutOfRangeException(nameof(kind)), + }; + BasePruner? pruner = kind == "tpe-pruned" ? new MedianPruner(nStartupTrials: 2) : null; + var study = Optuna.CreateStudy( + storage: storage, sampler: sampler, pruner: pruner, + directions: twoObjectives ? new[] { StudyDirection.Minimize, StudyDirection.Maximize } : Minimize); + int nTrials = kind switch + { + "gp" => 10, + "grid" => 6, + "bruteforce" => 12, + "tpe" or "tpe-independent" or "tpe-group" or "tpe-equal-choices" or "tpe-pruned" or "motpe-constrained" => 25, + _ => 40, + }; + + double[] Evaluate(Trial trial) + { + double x; + CategoricalChoice c; + if (kind == "bruteforce") + { + x = trial.SuggestFloat("x", -1, 1, step: 0.5); + trial.SuggestInt("n", 0, 1); + c = trial.SuggestCategorical("c", new CategoricalChoice[] { "a", true }); + } + else if (kind == "grid") + { + x = trial.SuggestFloat("x", -1, 3); + c = trial.SuggestCategorical("c", new CategoricalChoice[] { "a", 3L }); + } + else + { + x = trial.SuggestFloat("x", -5, 5); + if (kind != "gp") + { + trial.SuggestFloat("lx", 1e-3, 1e3, log: true); + trial.SuggestFloat("sx", -1, 1, step: 0.25); + trial.SuggestInt("ln", 1, 64, log: true); + } + trial.SuggestInt("n", -3, 3); + c = trial.SuggestCategorical("c", new CategoricalChoice[] { "a", "b", CategoricalChoice.None, 7L, 2.5 }); + if (kind == "tpe-group" && c.Equals((CategoricalChoice)"a")) + trial.SuggestFloat("only-a", 0, 1); + // Python-equal choices that are not identical (1 and 1.0): the search space keeps the first trial's + // distribution, which reads the other variant's compact trials through their external values. + if (kind == "tpe-equal-choices") + { + trial.SuggestCategorical("e", trial.Number % 2 == 0 + ? new CategoricalChoice[] { 1L, 2L } + : new CategoricalChoice[] { 1.0, 2.0 }); + } + } + trial.SetUserAttr("blob", new string('q', 90) + trial.Number); + trial.SetUserAttr("hex", LowerHex(65 + trial.Number % 3)); + trial.SetUserAttr("nested", new List { (long)trial.Number, "two", 3.5 }); + double value = x * x + (c.Value is string ? 0 : 1); + if (kind == "tpe-pruned") + { + for (int step = 0; step < 4; step++) + { + trial.Report(value + 1.0 / (step + 1), step); + if (trial.ShouldPrune()) + throw new TrialPruned(); + } + } + return twoObjectives ? new[] { value, -x } : new[] { value }; + } + + study.Optimize(t => (IReadOnlyList)Evaluate(t), nTrials: nTrials); + records = storage.GetAllTrials(study.StudyId, deepcopy: false); + // Fingerprints read only through the read-only accessors, so they cannot undo compaction themselves. + return records.Select(Fingerprint).ToList(); + } + + /// + /// A CMA-ES study stores its optimizer state as 2,045-digit lowercase hexadecimal chunks in system attributes; + /// the compact storage keeps them as bytes (two digits a byte), the sampler restores from them, and a second + /// process-free reload of the same storage continues to the same trials. + /// + [TestMethod] + public void CmaEsOptimizerState_IsStoredAsBytes_AndRestores() + { + static List Run(InMemoryStorage storage, int seed) + { + var study = Optuna.CreateStudy(storage: storage, sampler: new CmaEsSampler(seed: seed), studyName: "cma", + loadIfExists: true); + study.Optimize(t => Enumerable.Range(0, 6).Sum(i => Math.Pow(t.SuggestFloat($"x{i}", -3, 3) - 1, 2)), nTrials: 40); + return storage.GetAllTrials(study.StudyId, deepcopy: false).Select(Fingerprint).ToList(); + } + + var plain = new InMemoryStorage(); + var compact = new InMemoryStorage(compactFinishedTrials: true); + Run(compact, 2).Should().Equal(Run(plain, 2)); + // A new sampler over the same storage restores the optimizer from the stored chunks. + Run(compact, 3).Should().Equal(Run(plain, 3)); + + var holders = compact.GetAllTrials(0, deepcopy: false) + .Where(r => r.SystemAttrs.Keys.Any(k => k.StartsWith("cma:optimizer", StringComparison.Ordinal))) + .ToList(); + holders.Should().NotBeEmpty(); + foreach (FrozenTrial holder in holders) + { + var attrs = (CompactAttrs)holder.SystemAttrs; + foreach (string key in holder.SystemAttrs.Keys.Where(k => k.StartsWith("cma:optimizer", StringComparison.Ordinal))) + attrs.StoredValue(key).Should().BeOfType().Which.IsHex.Should().BeTrue(key); + } + } + + // ------------------------------------------------------------------ concurrency + + /// + /// Four workers optimize one compact study while other threads read every record through both kinds of + /// accessor: no read fails or sees a partial record, every finished record that only the read-only accessors + /// touched is still compact, and each record's contents never change between reads. + /// + [TestMethod] + public void ParallelWorkers_AndReaders_SeeConsistentRecords() + { + var storage = new InMemoryStorage(compactFinishedTrials: true); + var study = Optuna.CreateStudy(storage: storage, sampler: new TPESampler(seed: 7, nStartupTrials: 5)); + using var stop = new CancellationTokenSource(); + var failures = new List(); + var firstSeen = new System.Collections.Concurrent.ConcurrentDictionary(); + var readers = Enumerable.Range(0, 3).Select(r => Task.Run(() => + { + try + { + while (!stop.IsCancellationRequested) + { + foreach (FrozenTrial record in study.GetTrials(deepcopy: false)) + { + if (!record.State.IsFinished()) + continue; + string fingerprint = Fingerprint(record); + string first = firstSeen.GetOrAdd(record.Number, fingerprint); + if (first != fingerprint) + throw new InvalidOperationException($"trial {record.Number} changed between reads"); + // One reader also takes the mutable dictionaries of every third record. + if (r == 0 && record.Number % 3 == 0) + _ = record.ParamsDict.Count + record.UserAttrsDict.Count; + } + } + } + catch (Exception exception) + { + lock (failures) + failures.Add(exception); + } + })).ToArray(); + + study.Optimize(t => + { + t.SetUserAttr("blob", new string('w', 70)); + return Math.Pow(t.SuggestFloat("x", -2, 2), 2) + t.SuggestInt("n", 0, 3); + }, nTrials: 60, nJobs: 4); + stop.Cancel(); + Task.WaitAll(readers); + + failures.Should().BeEmpty(); + var records = storage.GetAllTrials(study.StudyId, deepcopy: false); + records.Should().HaveCount(60).And.OnlyContain(r => r.State == TrialState.Complete); + records.Where(r => r.Number % 3 != 0).Should().OnlyContain(r => r.IsCompact); + foreach (FrozenTrial record in records) + if (firstSeen.TryGetValue(record.Number, out string? seen)) + Fingerprint(record).Should().Be(seen); + } + + // ------------------------------------------------------------------ memory + + /// + /// What the switch is for: a finished 40-parameter random-search study retains at most a quarter of the memory + /// in the compact storage (measured about 0.76 KB against 6.8 KB a trial at 1,000 trials, 0.65 KB against + /// 6.7 KB at 5,000). + /// + [TestMethod] + [DoNotParallelize] + public void CompactStorage_RetainsAFractionOfTheMemory() + { + int verbosity = OptunaLogging.GetVerbosity(); + // Each finished trial logs a line with all its parameters at INFO, and the test host keeps every line it + // captures: about 2.3 KB a trial, more than the compact record itself, in both measurements. + OptunaLogging.SetVerbosity(LogLevel.Warning); + try + { + // A first study pays for the statics and the JIT of both paths outside the measurements. + RetainedBytes(compact: true, trials: 200); + long plain = RetainedBytes(compact: false, trials: 1000); + long compact = RetainedBytes(compact: true, trials: 1000); + compact.Should().BeLessThan(plain / 4, $"compact {compact / 1000.0:F0} KB against {plain / 1000.0:F0} KB"); + } + finally + { + OptunaLogging.SetVerbosity(verbosity); + } + } + + /// The heap a finished 40-parameter random-search study keeps alive. + /// Whether the storage compacts finished trials. + /// Number of trials to run. + /// Bytes retained by the storage and the study after a full collection. + [MethodImpl(MethodImplOptions.NoInlining)] + private static long RetainedBytes(bool compact, int trials) + { + string[] names = Enumerable.Range(0, 40).Select(i => "x" + i).ToArray(); + long before = GC.GetTotalMemory(forceFullCollection: true); + var storage = new InMemoryStorage(compactFinishedTrials: compact); + var study = Optuna.CreateStudy(storage: storage, sampler: new RandomSampler(seed: 0)); + study.Optimize(t => names.Sum(n => t.SuggestFloat(n, -10, 10)), nTrials: trials); + long after = GC.GetTotalMemory(forceFullCollection: true); + GC.KeepAlive(storage); + GC.KeepAlive(study); + return after - before; + } +} From f4eca0c5c781cd6cbdbf2ed8ccd615beecd8acd6 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 09:11:47 +0200 Subject: [PATCH 017/145] =?UTF-8?q?docs(plan13):=20name=20S12's=20commit?= =?UTF-8?q?=20(17716c7)=20in=20the=20status=20line=20and=20the=20=C2=A710?= =?UTF-8?q?=20row?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 17716c7 wrote "this commit" in the plan's §10 S12 row because a commit cannot know its own hash. This fills in the hash there and in the status line, and the row's peak working set now quotes the final proof run that §10.2's table reports (1,846 → 458 MB; the earlier figure, 1,845 → 460 MB, came from the second of three runs). No code change. --- docs/plans/13-rustuna-derived-performance.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plans/13-rustuna-derived-performance.md b/docs/plans/13-rustuna-derived-performance.md index 2af81f1..458fc69 100644 --- a/docs/plans/13-rustuna-derived-performance.md +++ b/docs/plans/13-rustuna-derived-performance.md @@ -2,7 +2,7 @@ > **Status: BUILT (2026-11-09).** Phases 1-4 are on `main`: `8f8ce3e` (S1-S3, D1), > `605806c` (S4, S7), `dce5885` (S3, S5, S6), `656ab94` (S10, S9 as measured). The owner then decided the three open -> items (2026-11-09): S8 built (`74893f7`), S12 built as an opt-in (`InMemoryStorage(compactFinishedTrials: true)`, +> items (2026-11-09): S8 built (`74893f7`), S12 built as an opt-in (`17716c7`, `InMemoryStorage(compactFinishedTrials: true)`, > §10.2), D2 not built (exact kernel only). S11 was measured and changes nothing. §10 records what was built, how it > was gated, and the before/after measurements. Background, benchmarks and the full Rustuna read-through: catalog note > `optunasharp/notes/rustuna-comparison-2026-11-08.md`. @@ -370,7 +370,7 @@ the `de495f8` snapshot and the working tree; and, for the concurrency-sensitive | S9 | `656ab94` | Measured first: attribute JSON is 3-4 % of a 10k-trial SQLite cold load, so lazy decoding was not built. Parsing each `distribution_json` row was ~40 %: both RDB storages now parse each distinct text once per load (`JournalDistributionCache`) | Cold load 2.1 → 1.4-1.5 s, 1.22 GB → 479 MB | | S11 | none | Measured: at 10-D and 40-D CMA-ES and 6-D GP the OpenBLAS thread count changes no result, but at 100-D every count gives a different (deterministic) result, as NumPy's does. Pinning one thread at `nJobs: 1` would break parity with Python's default-thread NumPy, so the automatic policy stays (pin only when `nJobs > 1`). One thread was 12 % faster at 40-D and 24-35 % at 100-D: available through `TrialExecutorOptions.BlasThreads = 1`, at the cost of parity | — | | S8 | `74893f7` | Owner 2026-11-09: "Build it", "Also have optuna's api overload". `SetTrialUserAttrs`/`SetTrialSystemAttrs`/`SetStudyUserAttrs`/`SetStudySystemAttrs` (virtual, default = one call per entry) plus Optuna-named overloads taking a dictionary; `Trial`/`Study.SetUserAttrs` and `SetUserAttr(dict)`. Overrides write in one step: one lock in memory, one transaction in SQLite and EF Core, one appended batch of single-key records in a journal (Python's replay asserts one key per record), with a drain so a rejected own record cannot poison the next call. Same result as the loop; a finished or unknown trial gets nothing written. Live Python reads a C#-written batch | — | -| S12 | this commit | Owner 2026-11-09: "Build it", opt-in if behavior changes. Built as `InMemoryStorage(compactFinishedTrials: true)`: results unchanged (every proof pair hash-identical, live-Optuna parity on it), but finished trials' read-only accessors return views and values are rebuilt per read, hence the switch. §10.2 | CMA-ES 100k: retained 1,287 → 257 MB, peak 1,845 → 460 MB | +| S12 | `17716c7` | Owner 2026-11-09: "Build it", opt-in if behavior changes. Built as `InMemoryStorage(compactFinishedTrials: true)`: results unchanged (every proof pair hash-identical, live-Optuna parity on it), but finished trials' read-only accessors return views and values are rebuilt per read, hence the switch. §10.2 | CMA-ES 100k: retained 1,287 → 257 MB, peak 1,846 → 458 MB | | D2 | not built | Owner 2026-11-09: "Keep exact only". TPE scores in NumPy's order with the S6 exact kernel; no fast-numerics path | — | **Findings along the way.** From adda96ca1b90f856e97c95c6df99fb1b14ea6017 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 13:07:38 +0200 Subject: [PATCH 018/145] perf(samplers): stop leaving NumSharp arrays to the finalizer on per-parameter paths (plan 14, step A); random search 5x, NSGA-II 1.9x, same trials The second Rustuna comparison (catalog note optunasharp/notes/rustuna-comparison-2026-11-08.md, "Second pass") found the biggest remaining cost was not a Rustuna technique at all: every NumSharp NDArray has a finalizer, and the hot sampler paths created arrays per suggestion and dropped them. Each one survived its first collection on the finalization queue and was promoted, so almost every gen-0 collection became a gen-1 (random 10k: gen-0/1 154/153; NSGA-II 10k: 87/86). GC pauses were 47 % of a 40-parameter random search and 41 % of NSGA-II. What changed (no result changes; every draw is the same call on the same generator, in the same order): - New Internal/NumPyDraws: Rand(), Rand(n), RandomSample(n) and Uniform(low, high) read the generator's bit generator directly (BitGenerator.NextDouble under BitGenerator.@lock, exactly what NumSharp's scalar and fill paths do: one next_double per value, MT19937's 27 + 26 bits of two consecutive words), so they create no array at all. Uniform keeps NumPy's order: range = high - low, the finiteness check before any draw (OverflowError text "Range exceeds valid bounds"), then low + range * U. RandInt (masked rejection) and Normal (cached polar Gaussian) still call NumSharp and dispose the 0-d/vector result. - SearchSpaceTransform: TransformValues (the managed array Transform wrapped) and UntransformValues(ReadOnlySpan) (the untransform without a NumSharp copy). Untransform(NDArray) now disposes its astype(float64) copy and delegates. The untransform result dictionary is sized up front. - RandomSampler.SampleIndependent draws the transform's bound rows one at a time through NumPyDraws.Uniform instead of rng.uniform(bounds[:, 0], bounds[:, 1]). Bit-identical: NumPy computes the same low + range * U per element from the same uniforms; the array form's only other difference (raising before any draw when a range is infinite) cannot arise, since a numerical parameter has one row and every categorical row is [0, 1]. It used to build five arrays per suggestion (bounds, two column views, the draws, the untransform copy). - NSGA-II family: NSGAIIChildGenerationStrategy (crossover decision, copy-parent pick, one decision per gene), NsgaiiCrossover (tournament randints, categorical uniform crossover, parents transformed straight from their params with TransformValues, child untransformed with UntransformValues, no population copy before the first pick), every crossover (Uniform, BLX-alpha, SBX/VSBX and their shared tail, SPX, UNDX) and PolynomialMutation go through NumPyDraws; result dictionaries are sized up front; NSGAIISampler.InferRelativeSearchSpace sizes its result. - CMA-ES: the per-ask seed draw, the told solutions' transform and the candidate's untransform use the helpers; the dimension comes from RawBounds instead of building the bounds array for its shape. QMC untransforms its point without a NumSharp copy. TPE's candidate sampling (mixture choice, categorical quantiles, truncated-normal quantiles) uses the helpers. Measured (proof harness of plan 13 built against the b3baef9 snapshot and this tree, 40 floats in [-10, 10], seed 0, warm rep, interleaved runs, machine at 25-48 % load from other sessions): random 10,000: 3.93-3.95 s -> 0.79-0.80 s (5.0x); allocated 2,305 -> 889 MB; gen-0/1/2 154/153/27 -> 56/16/7; GC pauses 1.67-1.69 -> 0.17-0.18 s; hash 5DC310069B28C5D7 both NSGA-II 10,000: 1.80-1.81 s -> 0.95-0.96 s (1.9x; Rustuna 0.2.1: 0.82 s); allocated 1,234 -> 962 MB; gen-0/1/2 87/86/20 -> 59/16/7; pauses 0.67 -> 0.20 s; hash 0B1FA7FBED2E9676 both CMA-ES 10,000 2.33 -> 2.20 s, TPE 1,000 3.80 -> 3.78 s, QMC 10,000 1.15 -> 1.21 s: unchanged within noise, hashes equal (their time is elsewhere; later plan 14 steps). Parity coverage (each affected area, byte for byte): - New unit NumPyDrawsTests: every helper against the NumSharp call it replaces, value bits and next stream state, on generators seeded alike (scalar/vector uniforms, TPE's sizes, every bound shape RandomSampler meets, row-by-row vs NumSharp's array uniform over a bounds column, non-finite ranges, randint seed ranges, Gaussians with the cached polar value, and NSGA-II's mixed draw sequence). - SearchSpaceTransformTests: TransformValues equals Transform and UntransformValues equals Untransform bit for bit (mixed space, with and without the unit cube, clamped and tied coordinates); the NDArray overload still reads float32/int64 input through a float64 copy and leaves the input usable. - RandomSamplerTests: 60 trials x 3 seeds over every distribution kind replay the pre-plan-14 array draw and match. - New live parity RandomSamplerParityTests.LongRun_AllDistributionKinds_IsBitIdentical_ToLiveOptuna: 300 trials x 2 seeds against live Optuna 5.0, one-hot categoricals with two and five mixed-type choices, log int to 2^40, stepped negative int, a float spanning +-1e300, every value compared through Python's repr. - Live parity suites rerun, all green against Optuna 5.0 (142/142, Slow long runs included): RandomSampler, NSGA-II (all crossovers, polynomial mutation, constraints, 2,000-trial 40-parameter run), NSGA-III, CMA-ES (+ mixed fleet), QMC, TPE (+ 1,000-trial and MOTPE 300-trial runs), compact storage, battle-tested usage, edge cases, partial fixed, brute force. - Hash probe (scratchpad p14/hashprobe, 35 seeded studies over every sampler family the change reaches directly or through RandomSampler and SearchSpaceTransform: random, TPE independent/multivariate/group/pruned/constrained, MOTPE, NSGA-II with every crossover, polynomial mutation and constraints, NSGA-III, SMS-EMOA, AGE-MOEA-2, MOEA/D, RVEA, CMA-ES plain/with margin/separable/IPOP, QMC Sobol/Halton, DE, PSO, SNES, xNES, OpenAI-ES, LM-CMA, VkD-CMA, JANUS, brute force): every hash equal before and after. - Unit suite, CI filter: net8.0 2136/2136; net10.0 2135/2136 with JournalConcurrencyTests.Lock_WithoutAGracePeriod_ WaitsForALeftBehindLockUntilItIsRemoved failing once under load and passing on 3 reruns (timing-sensitive journal lock test, untouched code). The OpenBugs subset of the touched areas fails the same 8 tests with the same messages before and after. - Documentation gate: check_documentation.cs 0 issues; Core builds with the doc warnings as errors (--no-incremental). --- src/OptunaSharp.Core/Internal/NumPyDraws.cs | 125 +++++++++ .../Internal/SearchSpaceTransform.cs | 60 +++- src/OptunaSharp.Core/Samplers/CmaEsSampler.cs | 13 +- .../Samplers/NSGAIISampler.cs | 7 +- .../Nsgaii/Crossovers/BLXAlphaCrossover.cs | 3 +- .../Nsgaii/Crossovers/SBXCrossover.cs | 3 +- .../Nsgaii/Crossovers/SPXCrossover.cs | 3 +- .../Samplers/Nsgaii/Crossovers/SbxLike.cs | 9 +- .../Nsgaii/Crossovers/UNDXCrossover.cs | 4 +- .../Nsgaii/Crossovers/UniformCrossover.cs | 2 +- .../Nsgaii/Crossovers/VSBXCrossover.cs | 7 +- .../Nsgaii/Mutations/PolynomialMutation.cs | 3 +- .../Nsgaii/NSGAIIChildGenerationStrategy.cs | 10 +- .../Samplers/Nsgaii/NsgaiiCrossover.cs | 33 +-- src/OptunaSharp.Core/Samplers/QMCSampler.cs | 5 +- .../Samplers/RandomSampler.cs | 26 +- .../Samplers/Tpe/ProbabilityDistributions.cs | 6 +- .../RandomSamplerParityTests.cs | 91 +++++++ .../Internal/NumPyDrawsTests.cs | 256 ++++++++++++++++++ .../Internal/SearchSpaceTransformTests.cs | 101 +++++++ .../Samplers/RandomSamplerTests.cs | 59 ++++ 21 files changed, 760 insertions(+), 66 deletions(-) create mode 100644 src/OptunaSharp.Core/Internal/NumPyDraws.cs create mode 100644 test/OptunaSharp.Tests/Internal/NumPyDrawsTests.cs diff --git a/src/OptunaSharp.Core/Internal/NumPyDraws.cs b/src/OptunaSharp.Core/Internal/NumPyDraws.cs new file mode 100644 index 0000000..6fbcce4 --- /dev/null +++ b/src/OptunaSharp.Core/Internal/NumPyDraws.cs @@ -0,0 +1,125 @@ +using System; +using NumSharp; + +namespace OptunaSharp.Internal; + +/// +/// NumPy random draws read into managed values without leaving a NumSharp array to the finalizer (plan 14). Use these +/// on per-trial and per-parameter paths instead of casting a draw ((double)rng.rand()): the draws, their order, +/// their bits and the generator's next state are exactly the cast's. +/// +/// +/// Why it matters. Every NumSharp carries a finalizer. One that is never disposed +/// survives its first garbage collection on the finalization queue and is promoted, so a hot path that drops arrays +/// turns almost every gen-0 collection into a gen-1 collection. Measured before plan 14 on 40-parameter studies: GC +/// pauses were 47 % of random search's time and 41 % of NSGA-II's, from exactly such per-parameter draws. +/// Two ways, one stream. The uniform draws (, , +/// , ) read the generator's bit generator directly, as NumSharp's own +/// scalar and fill paths do: one per value (NumPy's next_double, for +/// MT19937 the 27 + 26 bits of two consecutive words), under the bit generator's , so +/// no array is created at all. The draws whose algorithm carries more state ('s masked rejection, +/// 's cached polar Gaussian) call NumSharp and dispose the array it +/// returns. NumPyDrawsTests pins each helper against the NumSharp call it replaces, value and next +/// state, on generators seeded alike. +/// Concurrency. Each call takes the bit generator's lock for its own draws, as a NumSharp call does; a +/// helper never caches the bit generator, so a reseed (rng.seed(...)) is seen by the next call. +/// +internal static class NumPyDraws +{ + /// NumPy rng.rand() (random_sample()): one uniform draw in [0, 1) as a . + /// The generator to draw from; its state advances exactly as rng.rand() advances it. + /// The drawn value. + internal static double Rand(NumPyRandom rng) + { + BitGenerator bitGenerator = rng.get_bit_generator(); + lock (bitGenerator.@lock) + return bitGenerator.NextDouble(); + } + + /// NumPy rng.rand(n): uniform draws in [0, 1), in draw order. + /// The generator to draw from; its state advances exactly as rng.rand(n) advances it. + /// The number of draws; zero draws nothing and returns an empty array. + /// A new array holding the draws in the order NumPy fills them. + /// is negative (NumPy's negative-dimension error); nothing is drawn. + internal static double[] Rand(NumPyRandom rng, int n) => RandomSample(rng, n); + + /// NumPy rng.random_sample(n): uniform draws in [0, 1), in draw order. + /// The generator to draw from; its state advances exactly as rng.random_sample(n) advances it. + /// The number of draws; zero draws nothing and returns an empty array. + /// A new array holding the draws in the order NumPy fills them. + /// is negative (NumPy's negative-dimension error); nothing is drawn. + internal static double[] RandomSample(NumPyRandom rng, long n) + { + if (n < 0) + throw new ValueError("negative dimensions are not allowed"); + var draws = new double[n]; + BitGenerator bitGenerator = rng.get_bit_generator(); + // One lock for the whole fill, as NumSharp's random_sample(n) holds it: the n values are consecutive in the stream + // even when another thread shares the generator. + lock (bitGenerator.@lock) + { + for (long i = 0; i < n; i++) + draws[i] = bitGenerator.NextDouble(); + } + return draws; + } + + /// NumPy rng.randint(low, high): one integer draw in [, ). + /// The generator to draw from; its state advances exactly as rng.randint(low, high) advances it. + /// The smallest value that can be drawn. + /// One past the largest value that can be drawn. + /// The drawn value, as the 64-bit integer NumPy's default randint dtype holds. + /// Calls NumSharp's randint, whose masked rejection sampling consumes a data-dependent number of words, and disposes the 0-d array it returns. + /// is not greater than (NumPy's low >= high error). + internal static long RandInt(NumPyRandom rng, long low, long high) + { + using NDArray draw = rng.randint(low, high); + return (long)draw; + } + + /// NumPy rng.uniform(low, high) with scalar bounds: low + (high - low) * U for one uniform U. + /// The generator to draw from; its state advances exactly as rng.uniform(low, high) advances it. + /// The lower bound of the draw. + /// The upper bound of the draw. + /// The drawn value. + /// + /// NumPy's random_uniform, lower + range * next_double, with the range computed first and checked for + /// finiteness before anything is drawn, as both NumPy and NumSharp do. With one-element bound arrays NumPy computes the + /// same expression from the same uniform, so drawing a bounds column one row at a time with this helper reproduces + /// rng.uniform(bounds[:, 0], bounds[:, 1]) bit for bit, as long as no range is infinite (the array form raises + /// before its first draw, a row-by-row loop only when it reaches the bad row). + /// + /// high - low is infinite or NaN (NumPy's OverflowError('Range exceeds valid bounds')); nothing is drawn. + internal static double Uniform(NumPyRandom rng, double low, double high) + { + double range = high - low; + if (!double.IsFinite(range)) + throw new OverflowException("Range exceeds valid bounds"); + return low + range * Rand(rng); + } + + /// NumPy rng.normal(loc, scale): one Gaussian draw as a . + /// The generator to draw from; its state, including the cached second Gaussian of the polar method, advances exactly as rng.normal(loc, scale) advances it. + /// The mean of the distribution. + /// The standard deviation of the distribution. + /// The drawn value. + /// is negative (NumPy's scale < 0 error). + internal static double Normal(NumPyRandom rng, double loc, double scale) + { + using NDArray draw = rng.normal(loc, scale); + return (double)draw; + } + + /// NumPy rng.normal(loc, scale, size=n): Gaussian draws, in draw order. + /// The generator to draw from; its state advances exactly as rng.normal(loc, scale, n) advances it. + /// The mean of the distribution. + /// The standard deviation of the distribution. + /// The number of draws. + /// A new array holding the draws in the order NumPy fills them. + /// is negative, or is negative. + internal static double[] Normal(NumPyRandom rng, double loc, double scale, int n) + { + using NDArray draws = rng.normal(loc, scale, new Shape(n)); + return draws.ToArray(); + } +} diff --git a/src/OptunaSharp.Core/Internal/SearchSpaceTransform.cs b/src/OptunaSharp.Core/Internal/SearchSpaceTransform.cs index 43422a4..ed146c4 100644 --- a/src/OptunaSharp.Core/Internal/SearchSpaceTransform.cs +++ b/src/OptunaSharp.Core/Internal/SearchSpaceTransform.cs @@ -85,8 +85,23 @@ public NDArray Bounds /// Transform a parameter configuration from actual values to continuous space. /// /// A parameter configuration to transform. - /// A 1-dimensional float64 array holding the transformed parameters. + /// A 1-dimensional float64 array holding the transformed parameters; the caller owns it and should dispose it. + /// + /// A caller that only needs the values should call : this overload wraps the same + /// values in a NumSharp array, which waits for the finalizer when it is not disposed (plan 14). + /// + /// A numerical parameter value is not a real number. public NDArray Transform(IReadOnlyDictionary parameters) + => np.array(TransformValues(parameters)); + + /// + /// Transform a parameter configuration from actual values to continuous space, as a managed array: the values + /// returns, without the NumSharp array around them. + /// + /// A parameter configuration holding every parameter of the search space. + /// A new array of the transformed parameters, one entry per encoded column (one-hot columns for categoricals). + /// A numerical parameter value is not a real number. + internal double[] TransformValues(IReadOnlyDictionary parameters) { int n = _rawBounds.GetLength(0); var transParams = new double[n]; @@ -124,21 +139,44 @@ public NDArray Transform(IReadOnlyDictionary parameters) } } - return np.array(transParams); + return transParams; } /// /// Untransform a parameter configuration from continuous space to actual values. /// - /// A 1-dimensional array in the transformed space. + /// A 1-dimensional array in the transformed space; any numeric dtype, read as float64. /// An untransformed parameter configuration keyed by parameter name. + /// + /// Reads the values through NumPy's astype(float64) copy, which is disposed here, then untransforms them as + /// does. A caller holding a managed float64 array should call that + /// overload directly and skip the NumSharp copy. + /// public OrderedDict Untransform(NDArray transParams) { - int n = _rawBounds.GetLength(0); - System.Diagnostics.Debug.Assert(transParams.shape.Length == 1 && transParams.shape[0] == n); + System.Diagnostics.Debug.Assert(transParams.shape.Length == 1 && transParams.shape[0] == _rawBounds.GetLength(0)); + + // astype copies by default, so the copy is this method's to dispose; left alone it would survive a collection on + // the finalization queue (plan 14). + double[] values; + using (NDArray asFloat64 = transParams.astype(np.float64)) + values = asFloat64.ToArray(); + return UntransformValues(values); + } - var values = transParams.astype(np.float64).ToArray(); + /// + /// Untransform a parameter configuration from continuous space to actual values, reading float64 values directly: + /// the result gives for the same values, without a NumSharp array. + /// + /// One value per encoded column, in the transformed space; not modified. + /// An untransformed parameter configuration keyed by parameter name, in search-space order. + /// A distribution of the search space is not supported (cannot happen for a transform this class built). + internal OrderedDict UntransformValues(ReadOnlySpan transParams) + { + int n = _rawBounds.GetLength(0); + System.Diagnostics.Debug.Assert(transParams.Length == n); + ReadOnlySpan values = transParams; if (_transform01) { var scaled = new double[n]; @@ -147,7 +185,9 @@ public NDArray Transform(IReadOnlyDictionary parameters) values = scaled; } - var parameters = new OrderedDict(); + // Sized for the search space up front: the result is built on every untransform (per trial and per independent + // draw), and growing an OrderedDictionary from empty reallocates several times for a few dozen parameters. + var parameters = new OrderedDict(_searchSpace.Count); int column = 0; foreach (var (name, distribution) in _searchSpace) @@ -160,10 +200,10 @@ public NDArray Transform(IReadOnlyDictionary parameters) { // Select the highest rated one-hot encoding (np.argmax: first max wins). int argmax = 0; - double best = values[encodedColumns[0]]; + double best = values[(int)encodedColumns[0]]; for (int i = 1; i < encodedColumns.Length; i++) { - double v = values[encodedColumns[i]]; + double v = values[(int)encodedColumns[i]]; if (v > best) { best = v; @@ -174,7 +214,7 @@ public NDArray Transform(IReadOnlyDictionary parameters) } else { - param = UntransformNumericalParam(values[encodedColumns[0]], distribution, _transformLog); + param = UntransformNumericalParam(values[(int)encodedColumns[0]], distribution, _transformLog); } parameters[name] = param; diff --git a/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs b/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs index 1f91871..c41ff94 100644 --- a/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs +++ b/src/OptunaSharp.Core/Samplers/CmaEsSampler.cs @@ -310,7 +310,9 @@ public override OrderedDict InferRelativeSearchSpace(S var restoredOptimizer = RestoreOptimizer(); var optimizer = restoredOptimizer ?? InitOptimizer(trans, study.Direction, searchSpace); - int nDim = (int)trans.Bounds.shape[0]; + // trans.bounds has one row per encoded column; its row count is read from the raw bounds instead of building the + // NumSharp bounds array only for its shape (plan 14). + int nDim = trans.RawBounds.GetLength(0); if (optimizer.Dim != nDim) { if (_warnIndependentSampling) @@ -337,7 +339,7 @@ public override OrderedDict InferRelativeSearchSpace(S double[] x = optimizer is CmaWm ? ((System.Collections.IEnumerable)t.SystemAttrs["x_for_tell"]!) .Cast().Select(v => Convert.ToDouble(v)).ToArray() - : trans.Transform(t.Params).ToArray(); + : trans.TransformValues(t.Params); double y = study.Direction == StudyDirection.Minimize ? t.Value!.Value : -t.Value!.Value; solutions.Add((x, y)); } @@ -369,7 +371,7 @@ public override OrderedDict InferRelativeSearchSpace(S } // Caution: optimizer should update its seed value. - int seed = (int)_cmaRng.Rng.randint(1, 65536) + trial.Number; + int seed = (int)NumPyDraws.RandInt(_cmaRng.Rng, 1, 65536) + trial.Number; optimizer.ReseedRng(seed); double[] cmaParams; if (optimizer is CmaWm cmaWm) @@ -388,9 +390,8 @@ public override OrderedDict InferRelativeSearchSpace(S study.Storage.SetTrialSystemAttr(trial.TrialId, AttrKeyGeneration, optimizer.Generation); - // View the candidate zero-copy; Untransform copies it into a managed dict, so dispose here. - using NDArray nd = np.array(cmaParams, copy: false); - return trans.Untransform(nd); + // Untransform the managed candidate as it is: Optuna's trans.untransform(params), without a NumSharp copy. + return trans.UntransformValues(cmaParams); } private string AttrKeyGeneration => _attrPrefix + "generation"; diff --git a/src/OptunaSharp.Core/Samplers/NSGAIISampler.cs b/src/OptunaSharp.Core/Samplers/NSGAIISampler.cs index 5afd7f4..ff4c607 100644 --- a/src/OptunaSharp.Core/Samplers/NSGAIISampler.cs +++ b/src/OptunaSharp.Core/Samplers/NSGAIISampler.cs @@ -133,8 +133,11 @@ public override void ReseedRng() /// public override OrderedDict InferRelativeSearchSpace(Study study, FrozenTrial trial) { - var searchSpace = new OrderedDict(); - foreach (var (name, distribution) in _searchSpace.Calculate(study)) + var calculated = _searchSpace.Calculate(study); + // Sized for the whole calculated space: this runs on every trial, and growing the result from empty reallocated + // it several times for a few dozen parameters (plan 14). + var searchSpace = new OrderedDict(calculated.Count); + foreach (var (name, distribution) in calculated) { // The untransform of _SearchSpaceTransform does not assume a single value, so single // value objects are sampled via sample_independent, not sample_relative. diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/BLXAlphaCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/BLXAlphaCrossover.cs index d6faacf..8ba3532 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/BLXAlphaCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/BLXAlphaCrossover.cs @@ -1,5 +1,6 @@ using System; using NumSharp; +using OptunaSharp.Internal; namespace OptunaSharp.Samplers.Nsgaii; @@ -29,7 +30,7 @@ public override double[] Crossover( double[][] parentsParams, NumPyRandom rng, Study study, double[,] searchSpaceBounds) { int n = searchSpaceBounds.GetLength(0); - var r = rng.rand(n).ToArray(); + var r = NumPyDraws.Rand(rng, n); var child = new double[n]; for (int j = 0; j < n; j++) { diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SBXCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SBXCrossover.cs index 162c949..6c59048 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SBXCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SBXCrossover.cs @@ -1,5 +1,6 @@ using System; using NumSharp; +using OptunaSharp.Internal; namespace OptunaSharp.Samplers.Nsgaii; @@ -46,7 +47,7 @@ public override double[] Crossover( int n = searchSpaceBounds.GetLength(0); double eta = _eta ?? (study.IsMultiObjective() ? 20.0 : 2.0); - var us = rng.rand(n).ToArray(); + var us = NumPyDraws.Rand(rng, n); var c1 = new double[n]; var c2 = new double[n]; diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SPXCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SPXCrossover.cs index 652621b..91f2e2c 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SPXCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SPXCrossover.cs @@ -1,5 +1,6 @@ using System; using NumSharp; +using OptunaSharp.Internal; namespace OptunaSharp.Samplers.Nsgaii; @@ -43,7 +44,7 @@ public override double[] Crossover( } // rs = np.power(rng.rand(n), 1 / (np.arange(n) + 1)) (Equation (2)). - var rand = rng.rand(n).ToArray(); + var rand = NumPyDraws.Rand(rng, n); var rs = new double[n]; for (int i = 0; i < n; i++) rs[i] = Math.Pow(rand[i], 1.0 / (i + 1)); diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SbxLike.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SbxLike.cs index f61b57b..ec65c25 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SbxLike.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/SbxLike.cs @@ -1,4 +1,5 @@ using NumSharp; +using OptunaSharp.Internal; namespace OptunaSharp.Samplers.Nsgaii; @@ -29,9 +30,9 @@ internal static double[] CollapseTwoChildren( var child2 = new double[n]; for (int i = 0; i < n; i++) { - if ((double)rng.rand() < useChildGeneProb) + if (NumPyDraws.Rand(rng) < useChildGeneProb) { - if ((double)rng.rand() >= uniformCrossoverProb) + if (NumPyDraws.Rand(rng) >= uniformCrossoverProb) { child1[i] = c1[i]; child2[i] = c2[i]; @@ -44,7 +45,7 @@ internal static double[] CollapseTwoChildren( } else { - if ((double)rng.rand() >= uniformCrossoverProb) + if (NumPyDraws.Rand(rng) >= uniformCrossoverProb) { child1[i] = parent0[i]; child2[i] = parent1[i]; @@ -57,6 +58,6 @@ internal static double[] CollapseTwoChildren( } } - return (double)rng.rand() < 0.5 ? child1 : child2; + return NumPyDraws.Rand(rng) < 0.5 ? child1 : child2; } } diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UNDXCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UNDXCrossover.cs index 11ff05e..fa62022 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UNDXCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UNDXCrossover.cs @@ -66,8 +66,8 @@ public override double[] Crossover( // Draw order: etas (n gaussians) then xi (1 gaussian) — Optuna passes the *variance* as the // normal `scale` argument, replicated verbatim. - var etas = rng.normal(0.0, Math.Pow(sigmaEta, 2.0), new Shape(n)).ToArray(); - double xi = (double)rng.normal(0.0, Math.Pow(_sigmaXi, 2.0)); + var etas = NumPyDraws.Normal(rng, 0.0, Math.Pow(sigmaEta, 2.0), n); + double xi = NumPyDraws.Normal(rng, 0.0, Math.Pow(_sigmaXi, 2.0)); var es = OrthonormalBasisVectorToPsl(parentsParams, n); // Orthonormal basis orthogonal to psl. diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UniformCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UniformCrossover.cs index ecdbcdf..7aced4d 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UniformCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/UniformCrossover.cs @@ -33,7 +33,7 @@ public override double[] Crossover( int nParams = searchSpaceBounds.GetLength(0); // masks = (rng.rand(n_params) >= swapping_prob).astype(int) // child_params = parents_params[masks, range(n_params)] - var r = rng.rand(nParams).ToArray(); + var r = NumPyDraws.Rand(rng, nParams); var child = new double[nParams]; for (int j = 0; j < nParams; j++) { diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/VSBXCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/VSBXCrossover.cs index aee0095..846b38b 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/VSBXCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Crossovers/VSBXCrossover.cs @@ -1,5 +1,6 @@ using System; using NumSharp; +using OptunaSharp.Internal; namespace OptunaSharp.Samplers.Nsgaii; @@ -46,7 +47,7 @@ public override double[] Crossover( double eta = _eta ?? (study.IsMultiObjective() ? 20.0 : 2.0); const double eps = 1e-10; - var us = rng.rand(n).ToArray(); + var us = NumPyDraws.Rand(rng, n); var beta1 = new double[n]; var beta2 = new double[n]; for (int j = 0; j < n; j++) @@ -59,7 +60,7 @@ public override double[] Crossover( var p1 = parentsParams[1]; var c1 = new double[n]; - double u1 = (double)rng.rand(); + double u1 = NumPyDraws.Rand(rng); for (int j = 0; j < n; j++) { c1[j] = u1 <= 0.5 @@ -68,7 +69,7 @@ public override double[] Crossover( } var c2 = new double[n]; - double u2 = (double)rng.rand(); + double u2 = NumPyDraws.Rand(rng); for (int j = 0; j < n; j++) { c2[j] = u2 <= 0.5 diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/Mutations/PolynomialMutation.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/Mutations/PolynomialMutation.cs index 67da3f6..b3dbcb0 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/Mutations/PolynomialMutation.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/Mutations/PolynomialMutation.cs @@ -1,5 +1,6 @@ using System; using NumSharp; +using OptunaSharp.Internal; namespace OptunaSharp.Samplers.Nsgaii; @@ -27,7 +28,7 @@ public PolynomialMutation(double eta = 20.0) /// public override double Mutation(double param, NumPyRandom rng, Study study, double[] searchSpaceBounds) { - double u = (double)rng.rand(); + double u = NumPyDraws.Rand(rng); double lb = searchSpaceBounds[0]; double ub = searchSpaceBounds[1]; double width = ub - lb; diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs index 76acc38..3ecb689 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/NSGAIIChildGenerationStrategy.cs @@ -86,7 +86,7 @@ public NSGAIIChildGenerationStrategy( // We choose a child based on the specified crossover method. OrderedDict childParams; - if ((double)_rng.Rng.rand() < _crossoverProb) + if (NumPyDraws.Rand(_rng.Rng) < _crossoverProb) { childParams = NsgaiiCrossover.PerformCrossover( _crossover, study, parentPopulation, searchSpace, _rng.Rng, _swappingProb, dominates); @@ -94,8 +94,8 @@ public NSGAIIChildGenerationStrategy( else { int parentPopulationSize = parentPopulation.Count; - var parentParams = parentPopulation[(int)_rng.Rng.randint(0, parentPopulationSize)].Params; - childParams = new OrderedDict(); + var parentParams = parentPopulation[(int)NumPyDraws.RandInt(_rng.Rng, 0, parentPopulationSize)].Params; + childParams = new OrderedDict(searchSpace.Count); foreach (var name in searchSpace.Keys) childParams[name] = parentParams[name]; } @@ -103,10 +103,10 @@ public NSGAIIChildGenerationStrategy( int nParams = childParams.Count; double mutationProb = _mutationProb ?? 1.0 / Math.Max(1.0, nParams); - var @params = new OrderedDict(); + var @params = new OrderedDict(childParams.Count); foreach (var paramName in childParams.Keys) { - if ((double)_rng.Rng.rand() >= mutationProb) + if (NumPyDraws.Rand(_rng.Rng) >= mutationProb) { @params[paramName] = childParams[paramName]; } diff --git a/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs b/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs index c669d3f..0783e8b 100644 --- a/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs +++ b/src/OptunaSharp.Core/Samplers/Nsgaii/NsgaiiCrossover.cs @@ -81,7 +81,7 @@ internal static class NsgaiiCrossover OrderedDict numericalSearchSpace, SearchSpaceTransform? numericalTransform) { - var childParams = new OrderedDict(); + var childParams = new OrderedDict(categoricalSearchSpace.Count + numericalSearchSpace.Count); if (categoricalSearchSpace.Count > 0) { @@ -100,21 +100,18 @@ internal static class NsgaiiCrossover return childParams; // The following is applied only for numerical parameters. - var numKeys = numericalSearchSpace.Keys.ToList(); + // Optuna transforms each parent's numerical params ({p: parent.params[p] for p in numerical_search_space}). The + // transform reads exactly those names from the parent's own params, so it is given them directly: no per-parent + // dictionary copy and no NumSharp array per parent (plan 14), the same values in the same column order. var parentsNumerical = new double[parents.Count][]; for (int i = 0; i < parents.Count; i++) - { - var row = new OrderedDict(); - foreach (var k in numKeys) - row[k] = parents[i].Params[k]; - parentsNumerical[i] = numericalTransform.Transform(row).ToArray(); - } + parentsNumerical[i] = numericalTransform.TransformValues(parents[i].Params); var childNumericalArray = crossover.Crossover( parentsNumerical, rng, study, numericalTransform.RawBounds); - // View the crossover child zero-copy; Untransform copies it out, so dispose here. - using NDArray childNd = np.array(childNumericalArray, copy: false); - var childNumericalParams = numericalTransform.Untransform(childNd); + // Read the crossover child as it is: the untransform Optuna applies, without wrapping the managed child in a + // NumSharp array first. + var childNumericalParams = numericalTransform.UntransformValues(childNumericalArray); foreach (var (k, v) in childNumericalParams) childParams[k] = v; @@ -135,11 +132,15 @@ private static List SelectParents( NumPyRandom rng, Func, bool> dominates) { - var parents = new List(); + var parents = new List(crossover.NParents); var selectedIds = new HashSet(); for (int i = 0; i < crossover.NParents; i++) { - var remaining = parentPopulation.Where(t => !selectedIds.Contains(t.TrialId)).ToList(); + // Optuna's `[t for t in parent_population if t._trial_id not in parent_ids]`. Before the first pick nothing is + // excluded, so that list is the population itself, in its order: it is used as it is instead of being copied. + IReadOnlyList remaining = selectedIds.Count == 0 + ? parentPopulation + : parentPopulation.Where(t => !selectedIds.Contains(t.TrialId)).ToList(); var parent = SelectParent(study, remaining, rng, dominates); parents.Add(parent); selectedIds.Add(parent.TrialId); @@ -160,8 +161,8 @@ private static FrozenTrial SelectParent( Func, bool> dominates) { int populationSize = parentPopulation.Count; - var candidate0 = parentPopulation[(int)rng.randint(0, populationSize)]; - var candidate1 = parentPopulation[(int)rng.randint(0, populationSize)]; + var candidate0 = parentPopulation[(int)NumPyDraws.RandInt(rng, 0, populationSize)]; + var candidate1 = parentPopulation[(int)NumPyDraws.RandInt(rng, 0, populationSize)]; // TODO(ohta): Consider crowding distance. return dominates(candidate0, candidate1, study.Directions) ? candidate0 : candidate1; @@ -197,7 +198,7 @@ private static bool IsContained(OrderedDict @params, OrderedDic object?[] parent0, object?[] parentLast, NumPyRandom rng, double swappingProb, int nCategorical) { // masks = (rng.rand(n) >= swapping_prob).astype(int); child = parent_params[masks, range(n)]. - var r = rng.rand(nCategorical).ToArray(); + var r = NumPyDraws.Rand(rng, nCategorical); var child = new object?[nCategorical]; for (int j = 0; j < nCategorical; j++) child[j] = r[j] >= swappingProb ? parentLast[j] : parent0[j]; diff --git a/src/OptunaSharp.Core/Samplers/QMCSampler.cs b/src/OptunaSharp.Core/Samplers/QMCSampler.cs index 457ea15..c56878c 100644 --- a/src/OptunaSharp.Core/Samplers/QMCSampler.cs +++ b/src/OptunaSharp.Core/Samplers/QMCSampler.cs @@ -193,9 +193,8 @@ public override OrderedDict InferRelativeSearchSpace(S double[] qmcPoint = SampleQmc(study, searchSpace); var trans = new SearchSpaceTransform(combined, transform01: true); - // View the QMC point zero-copy; Untransform copies it into a managed dict, so dispose here. - using NDArray nd = np.array(qmcPoint, copy: false); - var sample = trans.Untransform(nd); + // Untransform the managed point as it is: Optuna's trans.untransform(sample), without a NumSharp copy. + var sample = trans.UntransformValues(qmcPoint); var result = new OrderedDict(); foreach (var (name, value) in sample) diff --git a/src/OptunaSharp.Core/Samplers/RandomSampler.cs b/src/OptunaSharp.Core/Samplers/RandomSampler.cs index c3f1339..b3b181d 100644 --- a/src/OptunaSharp.Core/Samplers/RandomSampler.cs +++ b/src/OptunaSharp.Core/Samplers/RandomSampler.cs @@ -1,5 +1,4 @@ using System; -using NumSharp; using OptunaSharp.Collections; using OptunaSharp.Distributions; using OptunaSharp.Internal; @@ -61,16 +60,29 @@ public override OrderedDict InferRelativeSearchSpace(S => new(); /// + /// + /// Optuna draws rng.uniform(trans.bounds[:, 0], trans.bounds[:, 1]) over the transform's bound rows (one row + /// for a numerical parameter, one per choice for a categorical one) and untransforms the result. This draws the + /// same rows one at a time through : each row is NumPy's low + (high - low) * U + /// from the same uniform in the same order, so the value and the generator's next state are Optuna's bit for bit + /// (pinned live by RandomSamplerParityTests). The array form's only other difference, raising before any draw + /// when some range is infinite, cannot arise here: a numerical parameter has one row and every categorical row is + /// [0, 1]. + /// Why not the array form (plan 14): it built five NumSharp arrays per suggestion (the bounds, two column + /// views, the draws and the untransform's float64 copy) and left them to the finalizer, which made GC pauses 47 % + /// of a 40-parameter random search. + /// public override object? SampleIndependent( Study study, FrozenTrial trial, string paramName, BaseDistribution paramDistribution) { - var searchSpace = new OrderedDict { [paramName] = paramDistribution }; + var searchSpace = new OrderedDict(1) { [paramName] = paramDistribution }; var trans = new SearchSpaceTransform(searchSpace); - var bounds = trans.Bounds; - // uniform(bounds[:, 0], bounds[:, 1]) — one MT19937 draw per bound row, in row order, - // exactly as NumPy broadcasts. - var transParams = _rng.Rng.uniform(bounds[":,0"], bounds[":,1"]); + // RawBounds is trans.bounds here: this transform does not rescale to the unit cube. + double[,] bounds = trans.RawBounds; + var transParams = new double[bounds.GetLength(0)]; + for (int row = 0; row < transParams.Length; row++) + transParams[row] = NumPyDraws.Uniform(_rng.Rng, bounds[row, 0], bounds[row, 1]); - return trans.Untransform(transParams)[paramName]; + return trans.UntransformValues(transParams)[paramName]; } } diff --git a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs index a6510e4..50b759a 100644 --- a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs +++ b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs @@ -207,7 +207,7 @@ internal MixtureOfProductDistribution(double[] weights, IReadOnlyList(); + var rndQuantile = NumPyDraws.Rand(rng.Rng, batchSize); int nChoices = cat.Weights.GetLength(1); for (int s = 0; s < batchSize; s++) { @@ -243,7 +243,7 @@ internal MixtureOfProductDistribution(double[] weights, IReadOnlyList(); + var quantiles = NumPyDraws.RandomSample(rng.Rng, total); var aFlat = new double[total]; var bFlat = new double[total]; @@ -324,7 +324,7 @@ private static int[] Choice(LazyRandomState rng, double[] weights, int size) for (int i = 0; i < n; i++) cdf[i] /= last; - var u = rng.Rng.random_sample((long)size).ToArray(); + var u = NumPyDraws.RandomSample(rng.Rng, size); var result = new int[size]; for (int j = 0; j < size; j++) result[j] = BisectRight(cdf, u[j]); diff --git a/test/OptunaSharp.Tests.Interop/RandomSamplerParityTests.cs b/test/OptunaSharp.Tests.Interop/RandomSamplerParityTests.cs index d60fd87..4f81497 100644 --- a/test/OptunaSharp.Tests.Interop/RandomSamplerParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/RandomSamplerParityTests.cs @@ -105,6 +105,97 @@ public void MixedSearchSpace_IsBitIdentical_ToLiveOptuna(int seed) } } + /// + /// Plan 14: now draws the transform's bound rows one at a time through + /// NumPyDraws.Uniform instead of NumPy's array form (and creates no NumSharp arrays). This pins a long run over + /// every distribution kind to live Optuna: one-hot categoricals with two choices and with five choices of mixed types + /// (str, int, float, None), a log int spanning 1..2^40, a stepped int with negative bounds and a float spanning + /// ±1e300, 300 trials per seed, every value compared through Python's repr (exact for floats). + /// + /// The seed both samplers start from. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(0)] + [DataRow(31337)] + public void LongRun_AllDistributionKinds_IsBitIdentical_ToLiveOptuna(int seed) + { + PythonSession.EnsureBitExactHostOrInconclusive(); + const int nTrials = 300; + string[] names = { "a", "b", "c", "d", "e", "f", "g", "h", "w" }; + + var study = Optuna.CreateStudy(sampler: new RandomSampler(seed)); + study.Optimize(t => + { + double a = t.SuggestFloat("a", -10, 10); + t.SuggestFloat("b", 1e-5, 1e-1, log: true); + t.SuggestFloat("c", 0.1, 0.9, step: 0.2); + t.SuggestInt("d", 1, 100); + t.SuggestInt("e", 1, 1L << 40, log: true); + t.SuggestInt("f", -20, 20, step: 5); + t.SuggestCategorical("g", new[] { true, false }); + t.SuggestCategorical("h", new OptunaSharp.Distributions.CategoricalChoice[] + { "x", 1L, 2.5, OptunaSharp.Distributions.CategoricalChoice.None, "y" }); + t.SuggestFloat("w", -1e300, 1e300); + return a; + }, nTrials: nTrials); + var csharp = study.Trials + .Select(t => string.Join(" ", names.Select(n => ReprOf(t.Params[n])))) + .ToList(); + + var python = new List(); + using (Py.GIL()) + { + using PyModule scope = Py.CreateScope(); + scope.Exec( + "import optuna\n" + + "optuna.logging.set_verbosity(optuna.logging.WARNING)\n" + + "def objective(t):\n" + + " a = t.suggest_float('a', -10, 10)\n" + + " t.suggest_float('b', 1e-5, 1e-1, log=True)\n" + + " t.suggest_float('c', 0.1, 0.9, step=0.2)\n" + + " t.suggest_int('d', 1, 100)\n" + + " t.suggest_int('e', 1, 2**40, log=True)\n" + + " t.suggest_int('f', -20, 20, step=5)\n" + + " t.suggest_categorical('g', [True, False])\n" + + " t.suggest_categorical('h', ['x', 1, 2.5, None, 'y'])\n" + + " t.suggest_float('w', -1e300, 1e300)\n" + + " return a\n" + + $"study = optuna.create_study(sampler=optuna.samplers.RandomSampler(seed={seed}))\n" + + $"study.optimize(objective, n_trials={nTrials})\n" + + $"rows = [' '.join(repr(t.params[n]) for n in {PyList(names)}) for t in study.trials]\n"); + using var rows = scope.Get("rows"); + long count = rows.Length(); + for (int i = 0; i < count; i++) + { + using var row = rows[i]; + python.Add(row.As()); + } + } + + python.Should().HaveCount(nTrials); + for (int i = 0; i < nTrials; i++) + csharp[i].Should().Be(python[i], $"seed {seed} trial {i} (a b c d e f g h w)"); + } + + /// Python's repr of a suggested external value, for the value kinds the long run suggests. + /// A trial's external parameter value. + /// The text Python's repr prints for the same value. + /// The value is of a kind the long run never suggests. + private static string ReprOf(object? value) => value switch + { + null => "None", + bool b => b ? "True" : "False", + long l => l.ToString(System.Globalization.CultureInfo.InvariantCulture), + double d => OptunaSharp.Internal.PythonRepr.FloatRepr(d), + string s => "'" + s + "'", + _ => throw new NotSupportedException(value.GetType().Name), + }; + + /// A Python list literal of the given names, for embedding in a script. + /// Parameter names without quotes or backslashes. + /// The list literal, for example ['a', 'b']. + private static string PyList(IEnumerable names) => "[" + string.Join(", ", names.Select(n => "'" + n + "'")) + "]"; + [TestMethod] [TestCategory("RequiresPython")] public void DistributionJson_IsByteIdentical_ToLiveOptuna() diff --git a/test/OptunaSharp.Tests/Internal/NumPyDrawsTests.cs b/test/OptunaSharp.Tests/Internal/NumPyDrawsTests.cs new file mode 100644 index 0000000..fcbaf3f --- /dev/null +++ b/test/OptunaSharp.Tests/Internal/NumPyDrawsTests.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using AwesomeAssertions; +using NumSharp; +using OptunaSharp.Internal; + +namespace OptunaSharp.Tests.Internal; + +/// +/// Plan 14: pins every helper to the NumSharp call it replaces. Each test drives two +/// generators seeded alike, one through the helper and one through the NumSharp call the sampler code made before plan +/// 14, and asserts the values are bit-identical and the streams stay in step afterwards (the next draws agree), so the +/// helper consumes exactly the words the call consumed. NumSharp's own calls are pinned to NumPy by its oracle; the +/// samplers built on the helpers are pinned to live Optuna by the interop parity suites. +/// +[TestClass] +public class NumPyDrawsTests +{ + /// Two generators seeded alike: the left is drawn through , the right through NumSharp. + /// The seed both generators start from. + /// The helper-side and the NumSharp-side generator. + private static (NumPyRandom Helper, NumPyRandom NumSharp) Pair(int seed) + => (np.random.RandomState(seed), np.random.RandomState(seed)); + + /// Asserts both generators continue with the same words: a long run of plain uniforms must agree bit for bit. + /// The generator drawn through the helpers. + /// The generator drawn through NumSharp. + private static void AssertInStep(NumPyRandom helper, NumPyRandom reference) + { + using NDArray expected = reference.random_sample(64L); + double[] want = expected.ToArray(); + for (int i = 0; i < want.Length; i++) + BitConverter.DoubleToInt64Bits(NumPyDraws.Rand(helper)).Should().Be(BitConverter.DoubleToInt64Bits(want[i]), + $"the streams must stay in step (draw {i} after the compared calls)"); + } + + /// A thousand scalar uniforms equal NumSharp's rand() bit for bit, and the streams stay in step. + /// The seed both generators start from. + [TestMethod] + [DataRow(0)] + [DataRow(42)] + [DataRow(2147483646)] + public void Rand_Scalar_MatchesNumSharpRand_AndKeepsTheStreamInStep(int seed) + { + var (helper, reference) = Pair(seed); + for (int i = 0; i < 1000; i++) + { + using NDArray expected = reference.rand(); + BitConverter.DoubleToInt64Bits(NumPyDraws.Rand(helper)).Should().Be(BitConverter.DoubleToInt64Bits((double)expected)); + } + AssertInStep(helper, reference); + } + + /// A vector of uniforms equals NumSharp's rand(n) element by element, for empty, single and long vectors. + /// The number of draws. + [TestMethod] + [DataRow(0)] + [DataRow(1)] + [DataRow(7)] + [DataRow(1000)] + public void Rand_Vector_MatchesNumSharpRand(int n) + { + var (helper, reference) = Pair(123); + using NDArray expected = reference.rand(n); + double[] actual = NumPyDraws.Rand(helper, n); + actual.Should().HaveCount(n); + double[] want = expected.ToArray(); + for (int i = 0; i < n; i++) + BitConverter.DoubleToInt64Bits(actual[i]).Should().Be(BitConverter.DoubleToInt64Bits(want[i]), $"value {i} of {n}"); + AssertInStep(helper, reference); + } + + /// + /// A vector of uniforms equals NumSharp's random_sample(n), at TPE's sizes (24 candidates; 40 parameters × 24 + /// candidates). + /// + /// The number of draws. + [TestMethod] + [DataRow(0L)] + [DataRow(24L)] + [DataRow(960L)] + public void RandomSample_MatchesNumSharpRandomSample(long n) + { + var (helper, reference) = Pair(20260830); + using NDArray expected = reference.random_sample(n); + double[] actual = NumPyDraws.RandomSample(helper, n); + double[] want = expected.ToArray(); + actual.Should().HaveCount(want.Length); + for (int i = 0; i < want.Length; i++) + BitConverter.DoubleToInt64Bits(actual[i]).Should().Be(BitConverter.DoubleToInt64Bits(want[i]), $"value {i}"); + AssertInStep(helper, reference); + } + + /// A negative count raises NumPy's ValueError before any draw, so the stream does not move. + [TestMethod] + public void RandomSample_Negative_RaisesValueError_AndDrawsNothing() + { + var (helper, reference) = Pair(5); + Action act = () => NumPyDraws.RandomSample(helper, -1); + act.Should().Throw(); + AssertInStep(helper, reference); + } + + /// + /// The bounds RandomSampler meets: plain, log-transformed and step-widened float and int bounds, one-hot categorical + /// rows, a zero-width range, reversed bounds and very wide ones. + /// + /// Pairs of (low, high). + private static IEnumerable<(double Low, double High)> Bounds() => new[] + { + (-10.0, 10.0), (Math.Log(1e-5), Math.Log(1e-1)), (0.1 - 0.1, 0.9 + 0.1), (0.5, 100.5), (Math.Log(0.5), Math.Log(128.5)), + (-2.5, 22.5), (0.0, 1.0), (3.25, 3.25), (5.0, -5.0), (-1e300, 1e300), (1e-300, 2e-300), (-0.0, 0.0), + }; + + /// Scalar uniforms over every bound shape RandomSampler meets equal NumSharp's scalar uniform(low, high). + /// The seed both generators start from. + [TestMethod] + [DataRow(0)] + [DataRow(99)] + public void Uniform_MatchesNumSharpScalarUniform(int seed) + { + var (helper, reference) = Pair(seed); + for (int round = 0; round < 50; round++) + { + foreach (var (low, high) in Bounds()) + { + using NDArray expected = reference.uniform(low, high); + BitConverter.DoubleToInt64Bits(NumPyDraws.Uniform(helper, low, high)) + .Should().Be(BitConverter.DoubleToInt64Bits((double)expected), $"uniform({low}, {high}), round {round}"); + } + } + AssertInStep(helper, reference); + } + + /// + /// Drawing a bounds column row by row equals NumSharp's array form uniform(bounds[:, 0], bounds[:, 1]), which + /// RandomSampler used before plan 14: four one-hot rows of a categorical, and the single row of a numerical parameter. + /// + [TestMethod] + public void Uniform_RowByRow_MatchesNumSharpArrayUniform_OverABoundsColumn() + { + double[,] categorical = { { 0, 1 }, { 0, 1 }, { 0, 1 }, { 0, 1 } }; + double[,] numerical = { { Math.Log(1e-3), Math.Log(10.0) } }; + foreach (double[,] bounds in new[] { categorical, numerical }) + { + var (helper, reference) = Pair(77); + for (int round = 0; round < 100; round++) + { + int rows = bounds.GetLength(0); + var lows = new double[rows]; + var highs = new double[rows]; + for (int r = 0; r < rows; r++) + { + lows[r] = bounds[r, 0]; + highs[r] = bounds[r, 1]; + } + using NDArray lowArray = np.array(lows); + using NDArray highArray = np.array(highs); + using NDArray expected = reference.uniform(lowArray, highArray); + double[] want = expected.ToArray(); + for (int r = 0; r < rows; r++) + BitConverter.DoubleToInt64Bits(NumPyDraws.Uniform(helper, bounds[r, 0], bounds[r, 1])) + .Should().Be(BitConverter.DoubleToInt64Bits(want[r]), $"row {r} of {rows}, round {round}"); + } + AssertInStep(helper, reference); + } + } + + /// A non-finite range raises NumPy's OverflowError text before any draw, as NumSharp's call does, and the stream does not move. + /// The lower bound. + /// The upper bound. + [TestMethod] + [DataRow(double.MinValue, double.MaxValue)] + [DataRow(0.0, double.PositiveInfinity)] + [DataRow(double.NaN, 1.0)] + public void Uniform_NonFiniteRange_RaisesOverflow_AndDrawsNothing(double low, double high) + { + var (helper, reference) = Pair(3); + Action numSharp = () => reference.uniform(low, high).Dispose(); + numSharp.Should().Throw().WithMessage("Range exceeds valid bounds"); + Action act = () => NumPyDraws.Uniform(helper, low, high); + act.Should().Throw().WithMessage("Range exceeds valid bounds"); + AssertInStep(helper, reference); + } + + /// Integer draws equal NumSharp's randint(low, high), including the seed ranges CMA-ES and GP draw from. + /// The smallest value that can be drawn. + /// One past the largest value that can be drawn. + [TestMethod] + [DataRow(0L, 2L)] + [DataRow(0L, 50L)] + [DataRow(1L, 65536L)] + [DataRow(1L, 2147483646L)] + public void RandInt_MatchesNumSharpRandint(long low, long high) + { + var (helper, reference) = Pair(11); + for (int i = 0; i < 500; i++) + { + using NDArray expected = reference.randint(low, high); + NumPyDraws.RandInt(helper, low, high).Should().Be((long)expected, $"draw {i}"); + } + AssertInStep(helper, reference); + } + + /// + /// Gaussians equal NumSharp's normal, scalar and vector, including the polar method's cached second value: + /// odd vector sizes, scalars and plain uniforms are interleaved so a cached Gaussian must be handed out on both sides. + /// + [TestMethod] + public void Normal_Scalar_And_Vector_MatchNumSharp_IncludingTheCachedGaussian() + { + var (helper, reference) = Pair(42); + for (int round = 0; round < 40; round++) + { + int n = 1 + 2 * (round % 5); + using NDArray expectedVector = reference.normal(0.5, 0.35, new Shape(n)); + double[] actual = NumPyDraws.Normal(helper, 0.5, 0.35, n); + double[] want = expectedVector.ToArray(); + for (int i = 0; i < n; i++) + BitConverter.DoubleToInt64Bits(actual[i]).Should().Be(BitConverter.DoubleToInt64Bits(want[i]), $"vector draw {i}, round {round}"); + + using NDArray expectedScalar = reference.normal(-1.0, 4.0); + BitConverter.DoubleToInt64Bits(NumPyDraws.Normal(helper, -1.0, 4.0)) + .Should().Be(BitConverter.DoubleToInt64Bits((double)expectedScalar), $"scalar draw, round {round}"); + + using NDArray expectedUniform = reference.rand(); + BitConverter.DoubleToInt64Bits(NumPyDraws.Rand(helper)) + .Should().Be(BitConverter.DoubleToInt64Bits((double)expectedUniform), $"uniform between normals, round {round}"); + } + AssertInStep(helper, reference); + } + + /// + /// Every helper in the order NSGA-II's child generation uses them (a crossover decision, a parent tournament, a + /// crossover vector, a resample, per-gene decisions), repeated, equals the same sequence of NumSharp calls. + /// + [TestMethod] + public void MixedSequence_OfEveryHelper_MatchesTheNumSharpSequence() + { + var (helper, reference) = Pair(2026); + for (int round = 0; round < 200; round++) + { + using (NDArray d = reference.rand()) + NumPyDraws.Rand(helper).Should().Be((double)d); + using (NDArray d = reference.randint(0, 20)) + NumPyDraws.RandInt(helper, 0, 20).Should().Be((long)d); + using (NDArray d = reference.rand(8)) + NumPyDraws.Rand(helper, 8).Should().Equal(d.ToArray()); + using (NDArray d = reference.uniform(-5.0, 5.0)) + NumPyDraws.Uniform(helper, -5.0, 5.0).Should().Be((double)d); + using (NDArray d = reference.random_sample(3L)) + NumPyDraws.RandomSample(helper, 3).Should().Equal(d.ToArray()); + } + AssertInStep(helper, reference); + } +} diff --git a/test/OptunaSharp.Tests/Internal/SearchSpaceTransformTests.cs b/test/OptunaSharp.Tests/Internal/SearchSpaceTransformTests.cs index 82083b0..7f7d968 100644 --- a/test/OptunaSharp.Tests/Internal/SearchSpaceTransformTests.cs +++ b/test/OptunaSharp.Tests/Internal/SearchSpaceTransformTests.cs @@ -163,4 +163,105 @@ public void Categorical_Untransform_Picks_First_ArgMax() trans.Untransform(NumSharp.np.array(new[] { 0.4, 0.4, 0.1 }))["c"] .Should().Be("a", "np.argmax returns the first maximum"); } + + /// + /// Plan 14: returns exactly the values + /// wraps in a NumSharp array, for the mixed space, with and without the unit-cube rescaling, over many random + /// configurations (one-hot categorical columns included). + /// + /// Whether the transform rescales numerical columns to the unit cube. + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void TransformValues_Equal_Transform_BitForBit(bool transform01) + { + var space = MixedSpace(); + var trans = new SearchSpaceTransform(space, transform01: transform01); + var rng = new Random(7); + string[] choices = { "u", "v", "w" }; + for (int round = 0; round < 200; round++) + { + var config = new OrderedDict + { + ["f"] = -5.0 + 10.0 * rng.NextDouble(), + ["fl"] = Math.Exp(Math.Log(1e-3) + (Math.Log(10.0) - Math.Log(1e-3)) * rng.NextDouble()), + ["fs"] = 0.25 * rng.Next(0, 5), + ["i"] = (long)rng.Next(1, 21), + ["il"] = (long)rng.Next(1, 65), + ["c"] = choices[rng.Next(0, 3)], + }; + using NumSharp.NDArray expected = trans.Transform(config); + double[] want = expected.ToArray(); + double[] actual = trans.TransformValues(config); + actual.Should().HaveCount(want.Length); + for (int i = 0; i < want.Length; i++) + BitConverter.DoubleToInt64Bits(actual[i]).Should().Be(BitConverter.DoubleToInt64Bits(want[i]), $"column {i}, round {round}"); + } + } + + /// + /// Plan 14: returns exactly what + /// returns for the same float64 values (same keys in the same order, same values and types), with and without the + /// unit-cube rescaling, including coordinates outside the bounds that the untransform clamps and one-hot ties. + /// + /// Whether the transform rescales numerical columns to the unit cube. + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void UntransformValues_Equal_Untransform_ForTheSameValues(bool transform01) + { + var trans = new SearchSpaceTransform(MixedSpace(), transform01: transform01); + double[,] bounds = trans.RawBounds; + int n = bounds.GetLength(0); + var rng = new Random(11); + for (int round = 0; round < 300; round++) + { + var values = new double[n]; + for (int i = 0; i < n; i++) + { + // Mostly inside the box, sometimes a little outside (the clamping branches), sometimes a tie (argmax). + double low = transform01 ? 0.0 : bounds[i, 0]; + double high = transform01 ? 1.0 : bounds[i, 1]; + values[i] = round % 7 == 0 ? 0.5 * (low + high) : low + (high - low) * (1.2 * rng.NextDouble() - 0.1); + } + using NumSharp.NDArray array = NumSharp.np.array(values); + var expected = trans.Untransform(array); + var actual = trans.UntransformValues(values); + actual.Keys.Should().Equal(expected.Keys); + foreach (var (name, value) in expected) + { + actual[name]!.GetType().Should().Be(value!.GetType(), $"{name}, round {round}"); + if (value is double d) + BitConverter.DoubleToInt64Bits((double)actual[name]!).Should().Be(BitConverter.DoubleToInt64Bits(d), $"{name}, round {round}"); + else + actual[name].Should().Be(value, $"{name}, round {round}"); + } + } + } + + /// + /// Plan 14: the NumSharp overload still reads any numeric dtype through astype(float64) (now disposing that copy): + /// a float32 and an int64 input untransform as their float64 conversions do, and the caller's array stays usable. + /// + [TestMethod] + public void Untransform_NonFloat64Input_IsReadThroughAFloat64Copy_AndTheInputSurvives() + { + var space = new OrderedDict + { + ["x"] = new FloatDistribution(-10.0, 10.0), + ["n"] = new IntDistribution(0, 9), + }; + var trans = new SearchSpaceTransform(space); + using NumSharp.NDArray asFloat32 = NumSharp.np.array(new[] { 2.5f, 3.4f }); + var fromFloat32 = trans.Untransform(asFloat32); + fromFloat32["x"].Should().Be((double)2.5f); + fromFloat32["n"].Should().Be(3L); + asFloat32.ToArray().Should().Equal(2.5f, 3.4f); + + using NumSharp.NDArray asInt64 = NumSharp.np.array(new[] { 4L, 7L }); + var fromInt64 = trans.Untransform(asInt64); + fromInt64["x"].Should().Be(4.0); + fromInt64["n"].Should().Be(7L); + asInt64.ToArray().Should().Equal(4L, 7L); + } } diff --git a/test/OptunaSharp.Tests/Samplers/RandomSamplerTests.cs b/test/OptunaSharp.Tests/Samplers/RandomSamplerTests.cs index b7166a3..a945f55 100644 --- a/test/OptunaSharp.Tests/Samplers/RandomSamplerTests.cs +++ b/test/OptunaSharp.Tests/Samplers/RandomSamplerTests.cs @@ -130,4 +130,63 @@ public void IntersectionSearchSpace_Rejects_Second_Study() act.Should().Throw() .WithMessage("`IntersectionSearchSpace` cannot handle multiple studies."); } + + /// + /// Plan 14: now draws the transform's bound rows one at a time instead + /// of NumPy's array form rng.uniform(trans.bounds[:, 0], trans.bounds[:, 1]). This replays the array form it + /// replaced, on a generator seeded alike, for every distribution kind (one-hot categoricals with two and five choices + /// of mixed types included) and asserts every suggested value is identical, so the generator also stays in step from + /// one suggestion to the next across a whole study. + /// + /// The sampler's seed, and the replay's. + [TestMethod] + [DataRow(0)] + [DataRow(123)] + [DataRow(20260830)] + public void SampleIndependent_RowByRow_Equals_TheArrayDrawItReplaced(int seed) + { + var distributions = new (string Name, BaseDistribution Distribution)[] + { + ("f", new FloatDistribution(-10, 10)), + ("fl", new FloatDistribution(1e-5, 1e-1, log: true)), + ("fs", new FloatDistribution(0.1, 0.9, step: 0.2)), + ("i", new IntDistribution(1, 100)), + ("il", new IntDistribution(1, 128, log: true)), + ("is", new IntDistribution(0, 20, step: 5)), + ("c2", new CategoricalDistribution(new CategoricalChoice[] { true, false })), + ("c5", new CategoricalDistribution(new CategoricalChoice[] { "a", 1L, 2.5, CategoricalChoice.None, "e" })), + ("wide", new FloatDistribution(-1e300, 1e300)), + }; + var study = Optuna.CreateStudy(sampler: new RandomSampler(seed)); + study.Optimize(t => + { + foreach (var (name, distribution) in distributions) + t.Suggest(name, distribution); + return 0.0; + }, nTrials: 60); + + var replay = new LazyRandomState(seed); + foreach (var trial in study.Trials) + { + foreach (var (name, distribution) in distributions) + { + // The pre-plan-14 body of SampleIndependent, verbatim apart from disposing its arrays. + var trans = new OptunaSharp.Internal.SearchSpaceTransform( + new OrderedDict { [name] = distribution }); + using NumSharp.NDArray bounds = trans.Bounds; + using NumSharp.NDArray lows = bounds[":,0"]; + using NumSharp.NDArray highs = bounds[":,1"]; + using NumSharp.NDArray draws = replay.Rng.uniform(lows, highs); + object? expected = trans.Untransform(draws)[name]; + if (expected is CategoricalChoice choice) + expected = choice.Value; + + object? actual = trial.Params[name]; + if (expected is double d) + BitConverter.DoubleToInt64Bits((double)actual!).Should().Be(BitConverter.DoubleToInt64Bits(d), $"trial {trial.Number} {name}"); + else + actual.Should().Be(expected, $"trial {trial.Number} {name}"); + } + } + } } From db4e46c7b765866f99c39b13d3ab44febf27fb11 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 13:44:27 +0200 Subject: [PATCH 019/145] perf(tpe): exact O(n) kernel widths, cached finished observations, no exp call where it is exactly zero (plan 14, step B); TPE 1.5x, MOTPE 1.2x, same trials Stopwatch timing of TPE's stages on 1,000 trials over 40 floats (the profiler's sampled thread time had hidden it: EventPipe suspensions inflate some frames) showed where a run went before this step: scoring the candidates 1.74 s (40 %), building the Parzen estimators 1.34 s (31 %; the bandwidth argsort alone 1.05 s, 24 %), converting the history to internal values 0.60 s (14 %). Three changes, each keeping every bit: 1. Kernel widths in O(n) when that is provably exact (ParzenEstimator.BinnedRawSigmas; Rustuna's binned_sigmas idea, rustuna_core/src/parzen_estimator/model.rs). A width is the larger distance to the two sorted neighbours, clipped up to minsigma, so a neighbour nearer than minsigma cannot change it. Binned at that width, a centre that is neither the smallest nor the largest of its bin clips to minsigma, and only each bin's extremes can have a farther neighbour (the adjacent non-empty bin's extreme, or the domain end). Rustuna trusts the binning and can differ in the last bits where rounding puts a centre in the neighbouring bin; this version proves its result instead: bin assignment floor((v - low) / minsigma) is monotone, so bins are contiguous runs of the sorted order and equal centres share a bin; a bin's smallest is taken at its first index and its largest at its last (the stable sort's positions); distances are the sort's subtractions of the same operands; and every bin is checked to span at most minsigma (computed), which by the monotonicity of IEEE subtraction bounds every in-bin neighbour distance, so dropping those changes no clipped width. Anything else (fewer than two centres, a centre outside the domain or NaN, a degenerate domain, magic clip off, a wider bin) falls back to the sort, now SortedRawSigmas. ComputeSigmas applies the same clip to either. 2. TpeObservationCache (Rustuna ddd9c56 / e7a22d7, "reuse completed observations across trials"): each finished trial's internal value per (parameter name, search-space distribution), by trial number, or the fact that it lacks the name. GetInternalRepr copies a known row from dense columns instead of chasing each trial's parameter dictionary; an unknown row is converted exactly as before (the subset test first, conversions in search-space order, so the same exceptions) and remembered. Unfinished trials (constant liar's running ones) are never cached. Columns are keyed by distribution value equality (equal distributions convert every value alike); a call for another study or storage starts over; past 256 columns (dynamic ranges) it starts over. Each instance (per-worker copies too) owns its cache under its own monitor. The non-cached path also reads each value with one TryGetValue instead of ContainsKey plus the indexer. 3. The log-sum-exp stores 0.0 for every term below MixtureOfProductDistribution.ExpIsZeroBelow (-746) instead of calling Math.Exp: exp there is exactly +0.0 (exp(-746) is 0.42 of half the smallest subnormal; NumPy's, ucrt's, glibc's and Apple's exp all return zero below the underflow point, about -745.1332), and the zero still enters the pairwise sum, so the summation tree is unchanged. Measured on this machine: Math.Exp costs 5 ns for normal results, 16 ns on its underflow-to-zero path and 63 ns where the result is subnormal (that band stays computed). Measured (plan 13 proof harness built against the b3baef9 snapshot and this tree, 40 floats in [-10, 10], seed 0, interleaved runs; hashes equal before and after): TPE 1,000 (warm rep): 3.81-3.95 s -> 2.51-2.55 s (1.5x); allocated 1,571 -> 1,028 MB; hash 77CB9F115180830A MOTPE 1,000 (rep 0): 12.67 s -> 10.68 s (1.19x); allocated 3,418 -> 2,832 MB; hash A14E3A50093C4DF4 MOTPE 300 (rep 0): 2.31 s -> 1.83 s before change 3 Stage timing after changes 1-2 (rep 0, scratch instrumented copy): internal values 0.60 -> 0.15 s, estimator construction 1.34 -> 0.54 s; scoring 1.75 s unchanged then (change 3 came after). MOTPE's scoring is dominated by exp calls whose results are subnormal (63 ns each), which an exact port cannot skip. Parity coverage (byte for byte where OptunaSharp is compared with itself; TPE vs live Optuna is trajectory parity within the documented erf tolerance, as before): - New TpeBinnedSigmasTests: binned vs sorted widths bit for bit over uniform, tied (grids), clustered, on-edge, one-ulp-off-edge, domain-end and floor-spaced centres, 14 sizes from 2 to 1,500, ten domains (log, step-widened, tiny, huge, far from zero), both endpoint settings, three seeds; asserts the fast path answers on more than 90 % of uniform inputs so the equality checks are not vacuous; fallback cases; ComputeSigmas end to end with and without the magic clip. Scratch fuzz: 800,000 more generated cases, about 769,000 answered by the binned path, 0 mismatches. - New TpeObservationCacheTests: warm cache vs the cache emptied before every suggestion, identical trials: dynamic ranges + conditional parameter + pruned trials in both modes on plain and compact storage, MOTPE, constant liar with running trials (ask three, tell three) in both modes, an ever-changing range (bounded at MaxColumns), and a sampler reused for a second study. - New TpeExpUnderflowTests: Math.Exp returns +0.0 bits for the 100,000 doubles below the bound and 200,000 inputs down to -1e6 (and -inf, double.MinValue) on every platform CI runs; the bound sits below the underflow point. - New live parity in TpeSamplerParityTests: DynamicRangesConditionalAndPrunedTrials (both modes, 4 cases, 80 trials) and ConstantLiarWithRunningTrials_AskTell (both modes, 3 cases, 15 rounds of 3 running trials). The whole TPE live suite passes, 103/103 with the importance, compact-storage, docs and recipe parity cases, Slow long runs included (TPE 1,000 trials x 40 parameters, MOTPE 300); importance parity (PED-ANOVA shares the estimator) 40/40. - Hash probe (35 seeded studies across every sampler family): 0 differences before/after. - Unit suite, CI filter: 2154/2154 on net8.0 and net10.0. Documentation gate: 0 issues; Core builds with the doc warnings as errors. --- src/OptunaSharp.Core/Samplers/TPESampler.cs | 105 ++++++++- .../Samplers/Tpe/ParzenEstimator.cs | 161 +++++++++++++- .../Samplers/Tpe/ProbabilityDistributions.cs | 20 +- .../Samplers/Tpe/TpeObservationCache.cs | 139 ++++++++++++ .../TpeSamplerParityTests.cs | 107 +++++++++ .../Samplers/TpeBinnedSigmasTests.cs | 188 ++++++++++++++++ .../Samplers/TpeExpUnderflowTests.cs | 49 ++++ .../Samplers/TpeObservationCacheTests.cs | 210 ++++++++++++++++++ 8 files changed, 958 insertions(+), 21 deletions(-) create mode 100644 src/OptunaSharp.Core/Samplers/Tpe/TpeObservationCache.cs create mode 100644 test/OptunaSharp.Tests/Samplers/TpeBinnedSigmasTests.cs create mode 100644 test/OptunaSharp.Tests/Samplers/TpeExpUnderflowTests.cs create mode 100644 test/OptunaSharp.Tests/Samplers/TpeObservationCacheTests.cs diff --git a/src/OptunaSharp.Core/Samplers/TPESampler.cs b/src/OptunaSharp.Core/Samplers/TPESampler.cs index 9a48079..c024644 100644 --- a/src/OptunaSharp.Core/Samplers/TPESampler.cs +++ b/src/OptunaSharp.Core/Samplers/TPESampler.cs @@ -65,6 +65,16 @@ public class TPESampler : BaseSampler /// The split cache, for the tests that count how many splits were computed rather than reused. internal TpeSplitCache SplitCache => _splitCache; + /// + /// Finished trials' internal parameter values, by trial number (plan 14): converts + /// each finished trial's value once instead of on every ask. Initialized here for the same reason as + /// : each instance, a per-worker copy included, owns its cache under its own monitor. + /// + private readonly TpeObservationCache _observationCache = new(); + + /// The observation cache, for the tests that check what it holds. + internal TpeObservationCache ObservationCache => _observationCache; + /// Deprecated (v4.3.0). Falls back to true. /// Deprecated (v4.9.0). Falls back to 1.0. /// Deprecated (v4.9.0). Falls back to true. @@ -432,6 +442,12 @@ private OrderedDict GetInternalRepr( } int rows = 0; + // A finished trial never changes, so its conversions are kept in the observation cache (plan 14): once each of + // this space's columns knows the trial, the row is copied from dense columns without touching the trial's + // parameters; a column that knows the trial lacks a name skips it, as the subset test below would. + TpeObservationCache.Column[] cached = _observationCache.ColumnsFor(study, names, distributions); + // The current trial's external values, read once each by name before any is converted (see below). + var externals = new object?[nParams]; // A compact finished trial (plan 13, item S12; InMemoryStorage built with compactFinishedTrials) stores each // value's internal form, exactly what ToInternalRepr would rebuild from its external one, so a parameter whose // search-space distribution converts as the trial's does is copied without boxing and converting it back. The @@ -440,6 +456,23 @@ private OrderedDict GetInternalRepr( int[] positions = Array.Empty(); foreach (var trial in trials) { + bool finished = trial.State.IsFinished(); + int number = trial.Number; + if (finished) + { + byte state = CachedRowState(cached, number); + if (state == TpeObservationCache.Absent) + continue; + if (state == TpeObservationCache.Present) + { + for (int i = 0; i < nParams; i++) + columns[i][rows] = cached[i].ValueOf(number); + rows++; + continue; + } + // Some column has not seen this trial yet: convert the whole row as before and remember it. + } + var @params = GetParams(trial, study); if (@params is CompactParams compact) { @@ -448,28 +481,49 @@ private OrderedDict GetInternalRepr( schema = compact.Schema; positions = schema.PositionsFor(names, distributions); } - if (Array.IndexOf(positions, ParamSchema.Absent) >= 0) + int absent = Array.IndexOf(positions, ParamSchema.Absent); + if (absent >= 0) + { + if (finished) + cached[absent].SetAbsent(number); continue; + } for (int i = 0; i < nParams; i++) { columns[i][rows] = positions[i] >= 0 ? compact.InternalAt(positions[i]) : distributions[i].ToInternalRepr(@params[names[i]]); } + if (finished) + RememberRow(cached, number, columns, rows); rows++; continue; } - // search_space.keys() <= params.keys(): the trial must carry every search-space param. - bool carriesAll = true; - for (int i = 0; i < nParams && carriesAll; i++) - carriesAll = @params.ContainsKey(names[i]); - if (carriesAll) + // search_space.keys() <= params.keys(): the trial must carry every search-space param. One lookup per name + // (plan 14): each value is read while checking its key, and the conversions start only once every key is + // known to be present, in search-space order, so a trial missing a name still converts nothing (and raises + // nothing), as with Optuna's subset test followed by the conversion loop. + int missing = -1; + for (int i = 0; i < nParams; i++) { - for (int i = 0; i < nParams; i++) - columns[i][rows] = distributions[i].ToInternalRepr(@params[names[i]]); - rows++; + if (!@params.TryGetValue(names[i], out externals[i])) + { + missing = i; + break; + } } + if (missing >= 0) + { + if (finished) + cached[missing].SetAbsent(number); + continue; + } + for (int i = 0; i < nParams; i++) + columns[i][rows] = distributions[i].ToInternalRepr(externals[i]); + if (finished) + RememberRow(cached, number, columns, rows); + rows++; } var result = new OrderedDict(nParams); @@ -478,6 +532,39 @@ private OrderedDict GetInternalRepr( return result; } + /// What the observation cache knows of one finished trial's row for a search space. + /// The search space's columns, in its order. + /// The finished trial's number. + /// + /// when some column knows the trial lacks its name (the row is skipped), + /// when every column holds its value, otherwise + /// (the row must be converted). + /// + private static byte CachedRowState(TpeObservationCache.Column[] cached, int number) + { + byte result = TpeObservationCache.Present; + for (int i = 0; i < cached.Length; i++) + { + byte state = cached[i].StateOf(number); + if (state == TpeObservationCache.Absent) + return TpeObservationCache.Absent; + if (state == TpeObservationCache.Unknown) + result = TpeObservationCache.Unknown; + } + return result; + } + + /// Stores a finished trial's freshly converted row in the observation cache, one value per column. + /// The search space's columns, in its order. + /// The finished trial's number. + /// The observation columns being built; row holds the trial's values. + /// The row just written. + private static void RememberRow(TpeObservationCache.Column[] cached, int number, double[][] columns, int row) + { + for (int i = 0; i < cached.Length; i++) + cached[i].SetPresent(number, columns[i][row]); + } + /// Python _sample. /// Study supplying trial history, objective directions, and storage. /// Trial snapshot being examined; copy its contents before modifying them. diff --git a/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs b/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs index bbe7691..875ca22 100644 --- a/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs +++ b/src/OptunaSharp.Core/Samplers/Tpe/ParzenEstimator.cs @@ -308,8 +308,52 @@ private static BatchedDistributions CalculateNumericalDistributions( /// Number of historical observations represented by the kernels. /// Kernel construction and observation-weight configuration. /// Per-observation kernel widths after endpoint and clipping heuristics. - private static double[] ComputeSigmas( + /// + /// The unclipped widths come from when it can prove they clip to exactly what Optuna's + /// sort gives (plan 14: the sort was a quarter of a 1,000-trial, 40-parameter TPE run), otherwise from + /// , Optuna's sort-based computation. Both then go through the same clip. + /// + internal static double[] ComputeSigmas( double[] mus, double low, double high, int nObservations, ParzenEstimatorParameters parameters) + { + int n = mus.Length; + double maxsigma = high - low; + double minsigma; + int nBins = 0; + if (parameters.ConsiderMagicClip) + { + int nKernels = nObservations + 1; // +1 for prior. + double divisor = Math.Min(100.0, 1.0 + nKernels); + minsigma = (high - low) / divisor; + // One bin per minsigma of the domain: (high - low) / minsigma is the divisor, a whole number from 3 to 100. + nBins = (int)divisor; + } + else + { + minsigma = Eps; + } + + double[] sigmas = (parameters.ConsiderMagicClip + ? BinnedRawSigmas(mus, low, high, minsigma, nBins, parameters.ConsiderEndpoints) + : null) + ?? SortedRawSigmas(mus, low, high, parameters.ConsiderEndpoints); + + for (int t = 0; t < n; t++) + sigmas[t] = MathUtil.Clip(sigmas[t], minsigma, maxsigma); + return sigmas; + } + + /// + /// Optuna's unclipped kernel widths, computed as compute_sigmas does: sort the centers, take each one's larger + /// distance to its sorted neighbors (the domain endpoints at both ends), and with consider_endpoints=False and + /// at least two centers drop the endpoint distance of the smallest and the largest. + /// + /// Kernel centers in transformed numerical coordinates; not modified. + /// Lower endpoint of the numerical domain. + /// Upper endpoint of the numerical domain. + /// Whether the domain endpoints count as neighbors of the smallest and largest center. + /// One unclipped width per center, in the centers' order. + internal static double[] SortedRawSigmas(double[] mus, double low, double high, bool considerEndpoints) { int n = mus.Length; @@ -330,7 +374,7 @@ private static double[] ComputeSigmas( for (int t = 0; t < n; t++) sortedSigmas[t] = Math.Max(swe[t + 1] - swe[t], swe[t + 2] - swe[t + 1]); - if (!parameters.ConsiderEndpoints && swe.Length >= 4) + if (!considerEndpoints && swe.Length >= 4) { sortedSigmas[0] = swe[2] - swe[1]; sortedSigmas[^1] = swe[^2] - swe[^3]; @@ -340,21 +384,116 @@ private static double[] ComputeSigmas( var sigmas = new double[n]; for (int t = 0; t < n; t++) sigmas[sortedIndices[t]] = sortedSigmas[t]; + return sigmas; + } - double maxsigma = high - low; - double minsigma; - if (parameters.ConsiderMagicClip) + /// + /// Unclipped kernel widths that clip to exactly what clips to, found in O(n) without + /// sorting, or when that cannot be proven for these centers (the caller then sorts). + /// + /// Kernel centers in transformed numerical coordinates; not modified. + /// Lower endpoint of the numerical domain. + /// Upper endpoint of the numerical domain. + /// The magic-clip floor every width is clipped up to; the bin width. + /// How many bins of width cover the domain. + /// Whether the domain endpoints count as neighbors of the smallest and largest center. + /// One width per center, in the centers' order, or . + /// + /// The idea is Rustuna's binned_sigmas (rustuna_core/src/parzen_estimator/model.rs): a width is the + /// larger distance to the two sorted neighbors, floored at minsigma by the clip, so a neighbor nearer than + /// minsigma cannot change it. Bin the centers at that width. A center that is neither the smallest nor the + /// largest of its bin has both sorted neighbors inside the bin, so its width clips to minsigma; only each + /// bin's extremes can have a farther neighbor, and that neighbor is the adjacent non-empty bin's extreme (or the + /// domain endpoint). Rustuna's version can differ from the sort in the last bits where rounding puts a center in + /// a neighboring bin; this one checks instead of trusting the binning. + /// Why the result is exact. Bin assignment floor((v - low) / minsigma) is monotone in + /// v (IEEE subtraction, division by a positive constant and floor all are), so the bins are contiguous runs + /// of the sorted order, and equal centers share a bin. A bin's smallest center is taken at its first index and its + /// largest at its last index, which are the positions a stable sort puts first and last among equal values; the + /// neighbor distances are the same subtractions of the same operands the sort makes. Every bin is verified to span + /// at most minsigma (max - min <= minsigma, computed); since IEEE subtraction is monotone in its + /// operands, every in-bin neighbor distance is then at most minsigma too, so leaving those distances out + /// changes no clipped width. Any bin wider than that, a center outside the domain or NaN, fewer than two + /// centers, or a degenerate domain returns . TpeBinnedSigmasTests compares the two + /// paths bit for bit over random, tied, clustered and bin-edge inputs. + /// + internal static double[]? BinnedRawSigmas( + double[] mus, double low, double high, double minsigma, int nBins, bool considerEndpoints) + { + int n = mus.Length; + if (n < 2 || nBins < 1 || nBins > 100 || !(high > low) || !(minsigma > 0) || !double.IsFinite(minsigma)) + return null; + + // Per bin: the smallest center and its first index, the largest and its last index (stable-sort positions). + Span binMin = stackalloc double[nBins]; + Span binMax = stackalloc double[nBins]; + Span binMinIndex = stackalloc int[nBins]; + Span binMaxIndex = stackalloc int[nBins]; + binMin.Fill(double.PositiveInfinity); + binMax.Fill(double.NegativeInfinity); + binMinIndex.Fill(-1); + binMaxIndex.Fill(-1); + for (int i = 0; i < n; i++) { - int nKernels = nObservations + 1; // +1 for prior. - minsigma = (high - low) / Math.Min(100.0, 1.0 + nKernels); + double v = mus[i]; + // Outside the domain (or NaN): the sort handles it; the bins assume every center lies in [low, high]. + if (!(v >= low && v <= high)) + return null; + int bin = (int)Math.Min(Math.Floor((v - low) / minsigma), nBins - 1); + if (v < binMin[bin]) + { + binMin[bin] = v; + binMinIndex[bin] = i; + } + if (v >= binMax[bin]) + { + binMax[bin] = v; + binMaxIndex[bin] = i; + } } - else + + // The proof's condition: every in-bin neighbor distance is at most minsigma, which holds when each bin's span is. + for (int b = 0; b < nBins; b++) { - minsigma = Eps; + if (binMinIndex[b] >= 0 && !(binMax[b] - binMin[b] <= minsigma)) + return null; } - for (int t = 0; t < n; t++) - sigmas[t] = MathUtil.Clip(sigmas[t], minsigma, maxsigma); + // Every center not at its bin's extremes clips to minsigma; start everyone there and raise the extremes. + var sigmas = new double[n]; + Array.Fill(sigmas, minsigma); + + // Each bin's smallest center: its sorted predecessor is the previous non-empty bin's largest (or `low`). + bool firstBin = true; + double previousMax = 0.0; + for (int b = 0; b < nBins; b++) + { + int i = binMinIndex[b]; + if (i < 0) + continue; + if (!firstBin) + sigmas[i] = Math.Max(sigmas[i], binMin[b] - previousMax); + else if (considerEndpoints) + sigmas[i] = Math.Max(sigmas[i], binMin[b] - low); + firstBin = false; + previousMax = binMax[b]; + } + + // Each bin's largest center: its sorted successor is the next non-empty bin's smallest (or `high`). + bool lastBin = true; + double nextMin = 0.0; + for (int b = nBins - 1; b >= 0; b--) + { + int i = binMaxIndex[b]; + if (i < 0) + continue; + if (!lastBin) + sigmas[i] = Math.Max(sigmas[i], nextMin - binMax[b]); + else if (considerEndpoints) + sigmas[i] = Math.Max(sigmas[i], high - binMax[b]); + lastBin = false; + nextMin = binMin[b]; + } return sigmas; } diff --git a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs index 50b759a..9fe9f65 100644 --- a/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs +++ b/src/OptunaSharp.Core/Samplers/Tpe/ProbabilityDistributions.cs @@ -166,6 +166,16 @@ internal BatchedDiscreteTruncLogNormDistributions(double[] mu, double[] sigma, d /// Python _MixtureOfProductDistribution. internal sealed class MixtureOfProductDistribution { + /// + /// A bound below which exp is exactly +0.0: exp(-746) is about 0.42 of half the smallest + /// subnormal (2^-1075, about 2.5e-324), so a correctly rounded result is zero, and NumPy's and .NET's exp + /// return zero for every input below the underflow point (about -745.1332; glibc, ucrt and Apple's libm alike). + /// stores 0.0 for such terms instead of calling , whose underflow + /// path costs about three normal calls; TpeExpUnderflowTests pins that returns + /// +0.0 below it on the platform that runs the tests. + /// + internal const double ExpIsZeroBelow = -746.0; + /// Nonnegative mixture-component probabilities, in shared kernel order. internal double[] Weights { get; } /// Coordinate distributions sharing the same mixture-component index. @@ -454,7 +464,15 @@ internal double[] LogPdf(double[,] x) if (double.IsNegativeInfinity(max)) max = 0.0; for (int k = 0; k < nKernels; k++) - expBuf[k] = Math.Exp(weightedLogPdf[s * nKernels + k] - max); + { + // exp(z) is exactly +0.0 for every z below the underflow point (about -745.13: the result is under half + // the smallest subnormal), so the call is skipped well below it (plan 14): kernels far from a candidate + // are most of a long history and each exp was a few nanoseconds of the hottest loop. The zero still + // goes into the buffer, so the pairwise sum sees the same elements; NaN fails the test and still + // reaches Math.Exp, as everything at or above the bound does. + double z = weightedLogPdf[s * nKernels + k] - max; + expBuf[k] = z < ExpIsZeroBelow ? 0.0 : Math.Exp(z); + } result[s] = Math.Log(MathUtil.PairwiseSum(expBuf)) + max; } return result; diff --git a/src/OptunaSharp.Core/Samplers/Tpe/TpeObservationCache.cs b/src/OptunaSharp.Core/Samplers/Tpe/TpeObservationCache.cs new file mode 100644 index 0000000..16174e2 --- /dev/null +++ b/src/OptunaSharp.Core/Samplers/Tpe/TpeObservationCache.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using OptunaSharp.Distributions; +using OptunaSharp.Storages; + +namespace OptunaSharp.Samplers.Tpe; + +/// +/// The internal value of each parameter TPE has read from a finished trial, by trial number (plan 14; Rustuna's +/// "reuse completed observations across trials", ddd9c56 / e7a22d7). Every ask converts the whole history once per +/// parameter (twice in the independent mode, for the below and the above group); a finished trial never changes, so +/// its conversion is done once and later asks read a dense column instead of chasing each trial's parameter +/// dictionary and boxed value. +/// +/// +/// Same values. A column is keyed by the parameter name and the search-space distribution the value is +/// converted with (value equality: equal distributions convert every external value alike), and holds exactly what +/// returned for that finished trial, or that the trial lacks the name. +/// Unfinished trials (constant liar's running trials, whose relative parameters still arrive) are never cached. +/// Lifetime. The cache answers for one study at a time: a call for another study or storage (a sampler +/// reused across studies) starts over. Dynamic search spaces add a column per distinct distribution; past +/// the cache starts over rather than grow without bound. +/// Concurrency. Not thread-safe; TPE uses it only inside sampling, which runs under the sampler's lock +/// (each instance has its own cache). +/// +internal sealed class TpeObservationCache +{ + /// How many (name, distribution) columns are kept before the cache starts over. + internal const int MaxColumns = 256; + + /// The state of a column entry whose trial has not been read yet. + internal const byte Unknown = 0; + + /// The state of a column entry holding the finished trial's internal value. + internal const byte Present = 1; + + /// The state of a column entry whose finished trial lacks the parameter. + internal const byte Absent = 2; + + private readonly Dictionary<(string Name, BaseDistribution Distribution), Column> _columns = new(); + private BaseStorage? _storage; + private int _studyId; + + /// One parameter's cached values by trial number, with each entry's state. + internal sealed class Column + { + private double[] _values = Array.Empty(); + private byte[] _states = Array.Empty(); + + /// The state of trial 's entry: , or . + /// The trial's number in its study. + /// The entry's state; beyond the entries written so far. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal byte StateOf(int number) => (uint)number < (uint)_states.Length ? _states[number] : Unknown; + + /// The internal value cached for trial ; meaningful only when its state is . + /// The trial's number in its study. + /// The cached internal value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal double ValueOf(int number) => _values[number]; + + /// Records the internal value a finished trial's parameter converts to. + /// The trial's number in its study. + /// The value returned. + internal void SetPresent(int number, double value) + { + EnsureCapacity(number); + _values[number] = value; + _states[number] = Present; + } + + /// Records that a finished trial lacks the parameter. + /// The trial's number in its study. + internal void SetAbsent(int number) + { + EnsureCapacity(number); + _states[number] = Absent; + } + + /// Grows both arrays (doubling) so is a valid index; new entries are . + /// The index that must become valid. + private void EnsureCapacity(int number) + { + if (number < _states.Length) + return; + int size = Math.Max(Math.Max(16, number + 1), 2 * _states.Length); + Array.Resize(ref _values, size); + Array.Resize(ref _states, size); + } + } + + /// How many (name, distribution) columns the cache holds now. + internal int ColumnCount => _columns.Count; + + /// + /// Forgets everything, as a new sampler would know nothing. Sampling never needs it (a changed study resets the cache + /// on its own); tests use it to compare a warm cache with a cold one. + /// + internal void Reset() + { + _columns.Clear(); + _storage = null; + _studyId = 0; + } + + /// + /// The columns for one search space, in its order, after starting over when the study changed or the cache grew past + /// . + /// + /// The study being sampled; its id and storage identify the history the cache answers for. + /// The search space's parameter names, in order. + /// The search space's distributions, in the same order. + /// One column per parameter, aligned with . + internal Column[] ColumnsFor(Study study, string[] names, BaseDistribution[] distributions) + { + if (!ReferenceEquals(_storage, study.Storage) || _studyId != study.StudyId) + { + _columns.Clear(); + _storage = study.Storage; + _studyId = study.StudyId; + } + if (_columns.Count + names.Length > MaxColumns) + _columns.Clear(); + + var result = new Column[names.Length]; + for (int i = 0; i < names.Length; i++) + { + var key = (names[i], distributions[i]); + if (!_columns.TryGetValue(key, out Column? column)) + { + column = new Column(); + _columns[key] = column; + } + result[i] = column; + } + return result; + } +} diff --git a/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs b/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs index 474332c..41f2a45 100644 --- a/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/TpeSamplerParityTests.cs @@ -405,4 +405,111 @@ public void LongRun_MultiObjective40D_300Trials_TrajectoryMatches() " return a, b\n" + "study.optimize(_obj, n_trials=300)\n"); } + + /// + /// Plan 14 gate: the observation cache keys each column by parameter name and distribution, and remembers that a + /// finished trial lacks a parameter. This study gives it every case: a float whose range changes with the trial + /// number (three distributions under one name), a log float suggested only on one categorical branch (absent from + /// the other trials), and pruned trials (finished, so cached, and part of TPE's history), in both TPE modes, for 80 + /// trials. The trajectory must stay in lock-step with live Optuna within the erf tolerance. + /// + /// Whether TPE models the parameters jointly (the dynamic range is then sampled independently). + /// The sampler seed on both sides. + /// Never escapes: the objective raises it to prune every seventh trial, and Optimize records that trial as pruned. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(false, 0)] + [DataRow(false, 42)] + [DataRow(true, 0)] + [DataRow(true, 7)] + public void DynamicRangesConditionalAndPrunedTrials_TrajectoryMatches(bool multivariate, int seed) + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), sampler: new TPESampler(seed: seed, multivariate: multivariate), studyName: "tpe"); + csStudy.Optimize(t => + { + double span = 5.0 + t.Number % 3; + double x = t.SuggestFloat("x", -span, span); + string c = t.SuggestCategorical("c", new[] { "a", "b", "c" }); + double y = c == "a" ? t.SuggestFloat("y", 1e-3, 10.0, log: true) : 0.0; + double value = x * x + y + (c == "b" ? 0.0 : 1.0); + t.Report(value, 0); + if (t.Number % 7 == 3) + throw new TrialPruned(); + return value; + }, nTrials: 80); + + AssertTrajectoryParity(csStudy, + "study = optuna.create_study(study_name='tpe', " + + $"sampler=optuna.samplers.TPESampler(seed={seed}, multivariate={(multivariate ? "True" : "False")}))\n" + + "def _obj(t):\n" + + " span = 5.0 + t.number % 3\n" + + " x = t.suggest_float('x', -span, span)\n" + + " c = t.suggest_categorical('c', ['a', 'b', 'c'])\n" + + " y = t.suggest_float('y', 1e-3, 10.0, log=True) if c == 'a' else 0.0\n" + + " value = x * x + y + (0.0 if c == 'b' else 1.0)\n" + + " t.report(value, 0)\n" + + " if t.number % 7 == 3:\n" + + " raise optuna.TrialPruned()\n" + + " return value\n" + + "study.optimize(_obj, n_trials=80)\n"); + } + + /// + /// Plan 14 gate: constant liar with trials still running. Trials are asked three at a time and told only after all + /// three have sampled, so each later sample sees the earlier ones RUNNING: their relative parameters come from + /// system attributes (multivariate) or their suggested values (independent), and the observation cache must leave + /// them uncached and read them fresh. Fifteen rounds, both TPE modes, in lock-step with live Optuna within the erf + /// tolerance. + /// + /// Whether TPE models the parameters jointly. + /// The sampler seed on both sides. + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow(false, 3)] + [DataRow(true, 3)] + [DataRow(true, 11)] + public void ConstantLiarWithRunningTrials_AskTell_TrajectoryMatches(bool multivariate, int seed) + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), + sampler: new TPESampler(seed: seed, multivariate: multivariate, constantLiar: true, nStartupTrials: 5), + studyName: "tpe"); + for (int round = 0; round < 15; round++) + { + var batch = new Trial[3]; + for (int i = 0; i < batch.Length; i++) + { + batch[i] = csStudy.Ask(); + batch[i].SuggestFloat("x", -5, 5); + batch[i].SuggestFloat("y", 1e-2, 1e2, log: true); + batch[i].SuggestCategorical("c", new[] { "u", "v" }); + } + foreach (var t in batch) + { + var p = csStudy.Storage.GetTrial(t.TrialId).Params; + double x = (double)p["x"]!, y = (double)p["y"]!; + csStudy.Tell(t, x * x + Math.Log(y) * Math.Log(y) + ((string)p["c"]! == "u" ? 0.0 : 0.5)); + } + } + + AssertTrajectoryParity(csStudy, + "import math\n" + + "study = optuna.create_study(study_name='tpe', " + + $"sampler=optuna.samplers.TPESampler(seed={seed}, multivariate={(multivariate ? "True" : "False")}, " + + "constant_liar=True, n_startup_trials=5))\n" + + "for _ in range(15):\n" + + " batch = [study.ask() for _ in range(3)]\n" + + " for t in batch:\n" + + " t.suggest_float('x', -5, 5)\n" + + " t.suggest_float('y', 1e-2, 1e2, log=True)\n" + + " t.suggest_categorical('c', ['u', 'v'])\n" + + " for t in batch:\n" + + " p = t.params\n" + + " study.tell(t, p['x'] ** 2 + math.log(p['y']) ** 2 + (0.0 if p['c'] == 'u' else 0.5))\n"); + } } diff --git a/test/OptunaSharp.Tests/Samplers/TpeBinnedSigmasTests.cs b/test/OptunaSharp.Tests/Samplers/TpeBinnedSigmasTests.cs new file mode 100644 index 0000000..73edcb2 --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/TpeBinnedSigmasTests.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using AwesomeAssertions; +using OptunaSharp.Internal; +using OptunaSharp.Samplers.Tpe; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// Plan 14: TPE's kernel widths come from whenever it can prove the result +/// equals Optuna's sort-based compute_sigmas (). These tests compare +/// the two paths bit for bit, after the shared clip, over many generated center sets: uniform, tied (integer grids, +/// repeated values), clustered, on and next to bin edges, at the domain ends, tiny and huge domains, with and without +/// endpoints; and they check the binned path really runs on ordinary inputs (a fallback-only implementation would pass +/// the equality tests vacuously). +/// +[TestClass] +public class TpeBinnedSigmasTests +{ + /// The clip applies to both paths' unclipped widths. + /// Unclipped widths; clipped in place. + /// The magic-clip floor. + /// The domain width. + /// The same array, clipped. + private static double[] Clip(double[] raw, double minsigma, double maxsigma) + { + for (int i = 0; i < raw.Length; i++) + raw[i] = MathUtil.Clip(raw[i], minsigma, maxsigma); + return raw; + } + + /// + /// Compares both paths for one center set: when the binned path answers, its clipped widths must equal the sort's + /// bit for bit. + /// + /// The centers. + /// The domain's lower end. + /// The domain's upper end. + /// Whether the endpoints count as neighbors. + /// Text identifying the case in a failure message. + /// Whether the binned path answered (false when it fell back). + private static bool CompareOnce(double[] mus, double low, double high, bool considerEndpoints, string label) + { + int n = mus.Length; + double divisor = Math.Min(100.0, 1.0 + (n + 1)); + double minsigma = (high - low) / divisor; + double maxsigma = high - low; + double[] expected = Clip(ParzenEstimator.SortedRawSigmas(mus, low, high, considerEndpoints), minsigma, maxsigma); + double[]? binned = ParzenEstimator.BinnedRawSigmas(mus, low, high, minsigma, (int)divisor, considerEndpoints); + if (binned is null) + return false; + Clip(binned, minsigma, maxsigma); + for (int i = 0; i < n; i++) + { + if (BitConverter.DoubleToInt64Bits(binned[i]) != BitConverter.DoubleToInt64Bits(expected[i])) + Assert.Fail($"{label}: width {i} of {n} is {binned[i]:R} binned, {expected[i]:R} sorted (center {mus[i]:R})"); + } + return true; + } + + /// The domains TPE meets: plain floats, log-transformed bounds, step-widened ints, unit cubes, tiny and huge spans. + /// (low, high) pairs. + private static IEnumerable<(double Low, double High)> Domains() => new[] + { + (-10.0, 10.0), (0.0, 1.0), (Math.Log(1e-5), Math.Log(1e-1)), (0.5, 100.5), (-0.5, 20.5), (Math.Log(0.5), Math.Log(128.5)), + (-1e300, 1e300), (1e-300, 3e-300), (123456.789, 123456.791), (-3.0, -2.999999), + }; + + /// + /// Uniform, tied, clustered, edge and endpoint center sets of many sizes over every domain, both endpoint settings: + /// whenever the binned path answers it equals the sort, and on uniform inputs it answers almost always. + /// + /// The generator seed for the center sets. + [TestMethod] + [DataRow(1)] + [DataRow(2)] + [DataRow(3)] + public void BinnedWidths_EqualSortedWidths_BitForBit(int seed) + { + var rng = new Random(seed); + int uniformCases = 0, uniformAnswered = 0, answered = 0, cases = 0; + int[] sizes = { 2, 3, 4, 5, 10, 25, 26, 97, 98, 99, 150, 500, 975, 1500 }; + foreach (var (low, high) in Domains()) + { + foreach (int n in sizes) + { + double divisor = Math.Min(100.0, 1.0 + (n + 1)); + double binWidth = (high - low) / divisor; + for (int kind = 0; kind < 7; kind++) + { + var mus = new double[n]; + for (int i = 0; i < n; i++) + { + double u = rng.NextDouble(); + mus[i] = kind switch + { + // Uniform over the domain. + 0 => low + (high - low) * u, + // A coarse grid: many exact ties, like integer or stepped parameters. + 1 => low + (high - low) * (rng.Next(0, 9) / 8.0), + // Three tight clusters: wide gaps between, tiny gaps within. + 2 => low + (high - low) * (0.1 + 0.4 * rng.Next(0, 3)) + binWidth * 1e-3 * u, + // Exactly on bin edges. + 3 => Math.Min(high, low + binWidth * rng.Next(0, (int)divisor + 1)), + // One ulp either side of a bin edge. + 4 => Math.Clamp(Nudge(low + binWidth * rng.Next(1, (int)divisor), rng), low, high), + // The domain ends, often repeated. + 5 => rng.Next(0, 3) switch { 0 => low, 1 => high, _ => low + (high - low) * u }, + // Two values a bin width apart, repeated: neighbor distances at the floor. + _ => low + binWidth * (rng.Next(0, 2) + 0.5 * (n % 2)), + }; + } + foreach (bool considerEndpoints in new[] { false, true }) + { + cases++; + bool ok = CompareOnce(mus, low, high, considerEndpoints, $"seed {seed} domain [{low:R}, {high:R}] n {n} kind {kind} endpoints {considerEndpoints}"); + if (ok) + answered++; + if (kind == 0) + { + uniformCases++; + if (ok) + uniformAnswered++; + } + } + } + } + } + + // A fallback-only implementation would pass the comparisons above vacuously: the fast path must actually run. + uniformAnswered.Should().BeGreaterThan(uniformCases * 9 / 10, "uniform centers almost never need the sort"); + answered.Should().BeGreaterThan(cases / 2); + } + + /// Moves a value by one or two ulps up or down. + /// The value to move. + /// Chooses the direction and the distance. + /// The neighboring double. + private static double Nudge(double value, Random rng) + { + double moved = rng.Next(0, 2) == 0 ? Math.BitDecrement(value) : Math.BitIncrement(value); + return rng.Next(0, 2) == 0 ? moved : (moved > value ? Math.BitIncrement(moved) : Math.BitDecrement(moved)); + } + + /// + /// The cases the binned path must hand back to the sort: fewer than two centers, a center outside the domain or NaN, + /// a degenerate domain, and a bin whose computed span exceeds the floor. + /// + [TestMethod] + public void BinnedWidths_FallBack_WhereTheProofDoesNotHold() + { + ParzenEstimator.BinnedRawSigmas(new[] { 0.5 }, 0.0, 1.0, 1.0 / 3.0, 3, false).Should().BeNull("one center"); + ParzenEstimator.BinnedRawSigmas(new[] { 0.5, 1.5 }, 0.0, 1.0, 0.25, 4, false).Should().BeNull("a center above the domain"); + ParzenEstimator.BinnedRawSigmas(new[] { -0.1, 0.5 }, 0.0, 1.0, 0.25, 4, false).Should().BeNull("a center below the domain"); + ParzenEstimator.BinnedRawSigmas(new[] { double.NaN, 0.5 }, 0.0, 1.0, 0.25, 4, false).Should().BeNull("NaN"); + ParzenEstimator.BinnedRawSigmas(new[] { 1.0, 1.0 }, 1.0, 1.0, 0.0, 4, false).Should().BeNull("an empty domain"); + // A bin can span a little more than its width only through rounding (or, as here, when the bins do not cover the + // domain and the last bin takes the rest); the guard refuses any span over the floor. Bins of width 0.1 but only + // four of them: 0.35 and 0.9 both land in the last bin, 0.55 apart. + ParzenEstimator.BinnedRawSigmas(new[] { 0.0, 0.35, 0.9 }, 0.0, 1.0, 0.1, 4, false).Should().BeNull("a bin spanning more than the floor"); + } + + /// + /// End to end through : the widths for a 975-center history (the size + /// of a 1,000-trial run's "above" group) equal the sort-based computation, with and without the magic clip (without + /// it the binned path is never used). + /// + /// Whether the magic clip is on (the only case the bins apply to). + /// Whether the endpoints count as neighbors. + [TestMethod] + [DataRow(true, false)] + [DataRow(true, true)] + [DataRow(false, false)] + public void ComputeSigmas_EqualsTheSortBasedComputation(bool considerMagicClip, bool considerEndpoints) + { + var rng = new Random(975); + var mus = new double[975]; + for (int i = 0; i < mus.Length; i++) + mus[i] = -5.0 + 10.0 * rng.NextDouble(); + var parameters = new ParzenEstimatorParameters(1.0, considerMagicClip, considerEndpoints, _ => Array.Empty(), false); + double[] actual = ParzenEstimator.ComputeSigmas(mus, -5.0, 5.0, mus.Length, parameters); + + double minsigma = considerMagicClip ? 10.0 / Math.Min(100.0, 1.0 + (mus.Length + 1)) : 1e-12; + double[] expected = Clip(ParzenEstimator.SortedRawSigmas(mus, -5.0, 5.0, considerEndpoints), minsigma, 10.0); + for (int i = 0; i < mus.Length; i++) + BitConverter.DoubleToInt64Bits(actual[i]).Should().Be(BitConverter.DoubleToInt64Bits(expected[i]), $"width {i}"); + } +} diff --git a/test/OptunaSharp.Tests/Samplers/TpeExpUnderflowTests.cs b/test/OptunaSharp.Tests/Samplers/TpeExpUnderflowTests.cs new file mode 100644 index 0000000..7081c74 --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/TpeExpUnderflowTests.cs @@ -0,0 +1,49 @@ +using System; +using AwesomeAssertions; +using OptunaSharp.Samplers.Tpe; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// Plan 14: TPE's log-sum-exp stores 0.0 instead of calling for every term below +/// . That is exact only if itself +/// returns +0.0 (the bits, not merely a value equal to zero) for every such input on the platform running the +/// sampler; this pins it on every platform CI runs the unit suite on. +/// +[TestClass] +public class TpeExpUnderflowTests +{ + /// + /// Every input from the bound down to -1e6, densely near the bound and spread below it, plus -inf, gives + /// bits 0 (+0.0). + /// + [TestMethod] + public void MathExp_IsPositiveZero_ForEveryInputBelowTheSkipBound() + { + double bound = MixtureOfProductDistribution.ExpIsZeroBelow; + // The first 100,000 doubles below the bound, one ulp apart. + double x = Math.BitDecrement(bound); + for (int i = 0; i < 100_000; i++, x = Math.BitDecrement(x)) + BitConverter.DoubleToInt64Bits(Math.Exp(x)).Should().Be(0L, $"exp({x:R})"); + // A spread of inputs down to -1e6. + var rng = new Random(746); + for (int i = 0; i < 200_000; i++) + { + double y = bound - Math.Pow(10, 6 * rng.NextDouble()); + BitConverter.DoubleToInt64Bits(Math.Exp(y)).Should().Be(0L, $"exp({y:R})"); + } + BitConverter.DoubleToInt64Bits(Math.Exp(double.NegativeInfinity)).Should().Be(0L); + BitConverter.DoubleToInt64Bits(Math.Exp(double.MinValue)).Should().Be(0L); + } + + /// + /// The bound sits below the underflow point, not above it: inputs just above -745.1332 still produce a nonzero + /// subnormal, and so must still go through . + /// + [TestMethod] + public void TheSkipBound_IsBelowTheUnderflowPoint() + { + Math.Exp(-745.13).Should().BeGreaterThan(0.0, "exp still has a subnormal result here"); + (MixtureOfProductDistribution.ExpIsZeroBelow < -745.14).Should().BeTrue(); + } +} diff --git a/test/OptunaSharp.Tests/Samplers/TpeObservationCacheTests.cs b/test/OptunaSharp.Tests/Samplers/TpeObservationCacheTests.cs new file mode 100644 index 0000000..c190788 --- /dev/null +++ b/test/OptunaSharp.Tests/Samplers/TpeObservationCacheTests.cs @@ -0,0 +1,210 @@ +using System; +using System.Collections.Generic; +using System.Text; +using AwesomeAssertions; +using OptunaSharp.Samplers; +using OptunaSharp.Samplers.Tpe; +using OptunaSharp.Storages; + +namespace OptunaSharp.Tests.Samplers; + +/// +/// Plan 14: TPE keeps finished trials' internal parameter values in a instead of +/// converting the whole history on every ask. Each test runs the same seeded study twice, once with the cache warm +/// (as sampling uses it) and once with it emptied before every suggestion (so every observation is converted from the +/// trial again, as before the cache), and asserts every trial is identical to the bit: dynamic ranges (one name, many +/// distributions), conditional parameters (finished trials lacking a name), pruned trials, constant liar with trials +/// still running, both TPE modes, two objectives and compact storage. +/// +[TestClass] +public class TpeObservationCacheTests +{ + /// Every trial's number, state, parameters (value bits) and values (bits), one line per trial. + /// The finished study. + /// The study's trials as text. + private static string Dump(Study study) + { + var sb = new StringBuilder(); + foreach (var t in study.Trials) + { + sb.Append(t.Number).Append(' ').Append(t.State).Append(' '); + foreach (var (k, v) in t.Params) + sb.Append(k).Append('=').Append(v is double d ? BitConverter.DoubleToInt64Bits(d).ToString("X16") : v?.ToString()).Append(';'); + if (t.Values is not null) + foreach (double v in t.Values) + sb.Append(' ').Append(BitConverter.DoubleToInt64Bits(v).ToString("X16")); + sb.Append('\n'); + } + return sb.ToString(); + } + + /// + /// The objective with a dynamic range, a conditional parameter and pruning; runs + /// before each suggestion (the cold run empties the cache there). + /// + /// The trial. + /// Called before each suggestion. + /// The objective value. + /// Every seventh trial is pruned after reporting its value. + private static double Objective(Trial t, Action beforeSuggest) + { + double span = 5.0 + t.Number % 3; + beforeSuggest(); + double x = t.SuggestFloat("x", -span, span); + beforeSuggest(); + string c = t.SuggestCategorical("c", new[] { "a", "b", "c" }); + double y = 0.0; + if (c == "a") + { + beforeSuggest(); + y = t.SuggestFloat("y", 1e-3, 10.0, log: true); + } + beforeSuggest(); + long k = t.SuggestInt("k", 0, 20, step: 2); + double value = x * x + y + k / 10.0 + (c == "b" ? 0.0 : 1.0); + t.Report(value, 0); + if (t.Number % 7 == 3) + throw new TrialPruned(); + return value; + } + + /// The study run warm and cold must be identical, for both TPE modes, with plain and compact storage. + /// Whether TPE models the parameters jointly. + /// Whether the storage keeps finished trials compact (plan 13, item S12). + [TestMethod] + [DataRow(false, false)] + [DataRow(true, false)] + [DataRow(false, true)] + [DataRow(true, true)] + public void WarmCache_EqualsColdCache_DynamicConditionalPruned(bool multivariate, bool compact) + { + string Run(bool cold) + { + var sampler = new TPESampler(seed: 21, multivariate: multivariate, nStartupTrials: 5); + var study = Optuna.CreateStudy(storage: new InMemoryStorage(compactFinishedTrials: compact), sampler: sampler); + Action before = cold ? () => sampler.ObservationCache.Reset() : () => { }; + study.Optimize(t => Objective(t, before), nTrials: 120); + return Dump(study); + } + + Run(cold: false).Should().Be(Run(cold: true)); + } + + /// Two objectives (MOTPE), warm and cold, must be identical. + [TestMethod] + public void WarmCache_EqualsColdCache_MultiObjective() + { + string Run(bool cold) + { + var sampler = new TPESampler(seed: 5, nStartupTrials: 5); + var study = Optuna.CreateStudy(sampler: sampler, directions: new[] { StudyDirection.Minimize, StudyDirection.Minimize }); + study.Optimize(t => + { + double a = 0, b = 0; + for (int i = 0; i < 6; i++) + { + if (cold) + sampler.ObservationCache.Reset(); + double v = t.SuggestFloat("x" + i, -3, 3); + a += v * v; + b += (v - 1) * (v - 1); + } + return new[] { a, b }; + }, nTrials: 100); + return Dump(study); + } + + Run(cold: false).Should().Be(Run(cold: true)); + } + + /// + /// Constant liar with trials still running (ask three, then tell three): running trials must never be cached, and + /// warm and cold runs must be identical in both TPE modes. + /// + /// Whether TPE models the parameters jointly. + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void WarmCache_EqualsColdCache_ConstantLiarWithRunningTrials(bool multivariate) + { + string Run(bool cold) + { + var sampler = new TPESampler(seed: 9, multivariate: multivariate, constantLiar: true, nStartupTrials: 4); + var study = Optuna.CreateStudy(sampler: sampler); + for (int round = 0; round < 25; round++) + { + var batch = new List(); + for (int i = 0; i < 3; i++) + { + var t = study.Ask(); + if (cold) + sampler.ObservationCache.Reset(); + t.SuggestFloat("x", -5, 5); + if (cold) + sampler.ObservationCache.Reset(); + t.SuggestCategorical("c", new[] { "u", "v", "w" }); + batch.Add(t); + } + foreach (var t in batch) + { + var p = study.Storage.GetTrial(t.TrialId).Params; + study.Tell(t, (double)p["x"]! * (double)p["x"]! + ((string)p["c"]! == "u" ? 0.0 : 1.0)); + } + } + return Dump(study); + } + + Run(cold: false).Should().Be(Run(cold: true)); + } + + /// + /// A range that changes on every trial makes a new column per distribution; the cache must stay within + /// by starting over, and the study must equal the cold run. + /// + [TestMethod] + public void EverChangingRange_StaysBounded_AndEqualsColdCache() + { + int maxSeen = 0; + string Run(bool cold) + { + var sampler = new TPESampler(seed: 4, nStartupTrials: 3); + var study = Optuna.CreateStudy(sampler: sampler); + study.Optimize(t => + { + if (cold) + sampler.ObservationCache.Reset(); + double x = t.SuggestFloat("x", 0, 1 + t.Number); + maxSeen = Math.Max(maxSeen, sampler.ObservationCache.ColumnCount); + return x; + }, nTrials: 400); + return Dump(study); + } + + Run(cold: false).Should().Be(Run(cold: true)); + maxSeen.Should().BeLessThanOrEqualTo(TpeObservationCache.MaxColumns); + } + + /// + /// One sampler reused for a second study (trial numbers start over) gives the second study exactly what a new sampler + /// with the same seed state would: the cache must start over when the study changes. + /// + [TestMethod] + public void SecondStudy_StartsTheCacheOver() + { + var reused = new TPESampler(seed: 8, nStartupTrials: 3); + var first = Optuna.CreateStudy(sampler: reused); + first.Optimize(t => Math.Pow(t.SuggestFloat("x", -2, 2) - 1, 2), nTrials: 40); + var second = Optuna.CreateStudy(sampler: reused); + second.Optimize(t => Math.Pow(t.SuggestFloat("x", -2, 2) + 1, 2), nTrials: 40); + + // The same sequence with the cache emptied before every suggestion: identical, so nothing carried over. + var cold = new TPESampler(seed: 8, nStartupTrials: 3); + var coldFirst = Optuna.CreateStudy(sampler: cold); + coldFirst.Optimize(t => { cold.ObservationCache.Reset(); return Math.Pow(t.SuggestFloat("x", -2, 2) - 1, 2); }, nTrials: 40); + var coldSecond = Optuna.CreateStudy(sampler: cold); + coldSecond.Optimize(t => { cold.ObservationCache.Reset(); return Math.Pow(t.SuggestFloat("x", -2, 2) + 1, 2); }, nTrials: 40); + + Dump(first).Should().Be(Dump(coldFirst)); + Dump(second).Should().Be(Dump(coldSecond)); + } +} From a2792548685fccac6af24c0ee6982f611c68afb7 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 13:51:28 +0200 Subject: [PATCH 020/145] perf(qmc): compute the sample-id key once per search space (plan 14, step C); QMC 2.4x, same trials QMCSampler finds its sample id in a study system attribute named "qmc:" + sha256(json.dumps(qmc_vars)), where qmc_vars holds the engine kind, the search space as str(distribution) by sorted name, and the scramble flag and seed (Optuna's _find_sample_id). OptunaSharp rebuilt that key on every trial: 40 distribution reprs (each an AsDict, a sort and a StringBuilder), a JSON document and a SHA-256. An allocation trace of 5,000 40-parameter trials put most of QMC's 1.58 GB per 10,000 trials there. The key is a pure function of the search space and the sampler's fixed settings (qmc type, scramble, seed are all readonly). QmcIdKey now keeps it for the last search space and reuses it while the next call passes the same names in the same order with the very same distribution objects, which a study's relative search space does from its first trial on (QMC builds it from the first trial's distributions). Reference identity, not value equality, decides: two categoricals can be equal by Python's rules (True == 1) and still print differently, so only the same objects are known to print the same; anything else recomputes. The sample-id read and write still happen on every call. The memo runs under the sampler's lock with the rest of sampling, and per-worker instances start without one. Also sized up front: the relative search space and the untransformed result dictionaries. Measured (plan 13 proof harness against the b3baef9 snapshot, 40 floats, seed 0, warm rep, interleaved): QMC 10,000: 1.24-1.30 s -> 0.52-0.53 s (2.4x); allocated 1,584 -> 472 MB; gen-0/1/2 98/22/10 -> 30/12/4; hash 1ADA76D940EDB34F both. Parity coverage: - New live gate QmcSamplerParityTests.LongRun_40Parameters_300Trials_FullState_IsBitIdentical: Sobol and Halton, scrambled and not, 37 floats + a log float + an int + a categorical over 300 trials; the whole study state, the qmc: counter in the study system attrs included, bit-identical to live Optuna 5.0. The QMC live suite passes, 16/16; with battle-tested usage and the recipes 47/47 before the new gate. - New unit test QmcSamplerTests.QmcIdKey_Memoized_EqualsAFreshComputation: a memoized key equals a fresh sampler's for the same objects, for equal-by-value categoricals that print differently (True/False vs 1/0, which must and do give a different key), for a reordered space, and back again; scrambled and not. - QMC unit tests 31/31. --- src/OptunaSharp.Core/Samplers/QMCSampler.cs | 84 ++++++++++++++++--- .../QmcSamplerParityTests.cs | 47 +++++++++++ test/OptunaSharp.Tests/QmcSamplerTests.cs | 34 ++++++++ 3 files changed, 154 insertions(+), 11 deletions(-) diff --git a/src/OptunaSharp.Core/Samplers/QMCSampler.cs b/src/OptunaSharp.Core/Samplers/QMCSampler.cs index c56878c..6d1524c 100644 --- a/src/OptunaSharp.Core/Samplers/QMCSampler.cs +++ b/src/OptunaSharp.Core/Samplers/QMCSampler.cs @@ -36,6 +36,15 @@ public class QMCSampler : BaseSampler private readonly bool _scramble; private readonly long _seed; + + /// The names of the search space was computed for, in order (plan 14). + private string[]? _lastKeyNames; + + /// The distribution objects of that search space, compared by reference. + private BaseDistribution[]? _lastKeyDistributions; + + /// The last sample-id attribute key computed; before the first relative sample. + private string? _lastKeyQmcId; private readonly BaseSampler _independentSampler; private readonly bool _warnIndependentSampling; private readonly string _qmcType; @@ -146,8 +155,9 @@ public override OrderedDict InferRelativeSearchSpace(S } // If an initial trial was already made, construct the search space from the first trial. - var result = new OrderedDict(); - foreach (var (name, dist) in firstTrial.Distributions) + var firstDistributions = firstTrial.Distributions; + var result = new OrderedDict(firstDistributions.Count); + foreach (var (name, dist) in firstDistributions) result[name] = dist; return result; } @@ -177,7 +187,7 @@ public override OrderedDict InferRelativeSearchSpace(S // [0, C), and int(value) (a floor, since the engines emit values in [0, 1)) gives a uniform // choice index in [0, C - 1]. IntDistribution would round to nearest and bias the boundaries. var categoricalSpace = new OrderedDict(); - var combined = new OrderedDict(); + var combined = new OrderedDict(searchSpace.Count); foreach (var (name, dist) in searchSpace) { if (dist is CategoricalDistribution cat) @@ -196,7 +206,7 @@ public override OrderedDict InferRelativeSearchSpace(S // Untransform the managed point as it is: Optuna's trans.untransform(sample), without a NumSharp copy. var sample = trans.UntransformValues(qmcPoint); - var result = new OrderedDict(); + var result = new OrderedDict(sample.Count); foreach (var (name, value) in sample) { if (categoricalSpace.TryGetValue(name, out var cat)) @@ -253,8 +263,52 @@ private double[] SampleQmc(Study study, OrderedDict se /// The sequence index allocated to this trial. private int FindSampleId(Study study, OrderedDict searchSpace) { + string keyQmcId = QmcIdKey(searchSpace); + + var studyAttrs = study.Storage.GetStudySystemAttrs(study.StudyId); + int sampleId = (studyAttrs.TryGetValue(keyQmcId, out var v) && v is not null + ? Convert.ToInt32(v) + : -1) + 1; + study.Storage.SetStudySystemAttr(study.StudyId, keyQmcId, sampleId); + return sampleId; + } + + /// + /// The study system attribute holding this engine's last sample id for : Optuna's + /// "qmc:" + sha256(json.dumps(qmc_vars)), where qmc_vars holds the engine kind, the search space as + /// str(distribution) by sorted name, and the scramble flag and seed. + /// + /// The relative search space being sampled. + /// The attribute key, qmc: followed by 64 lowercase hex digits. + /// + /// The key is a pure function of the search space and this sampler's fixed settings, and it was about a quarter of a + /// 40-parameter QMC trial (each distribution's repr, a JSON document and a hash, per trial). It is kept for the last + /// search space and reused while the next call passes the same names with the very same distribution objects (plan + /// 14), which a study's relative search space does from its first trial on. Reference identity, not value equality, + /// decides: two categoricals can be equal by Python's rules (True == 1) and still print differently, so + /// only the same objects are known to print the same. Runs under the sampler's lock, like all of sampling. + /// + internal string QmcIdKey(OrderedDict searchSpace) + { + if (_lastKeyQmcId is not null && _lastKeyNames!.Length == searchSpace.Count) + { + bool same = true; + int i = 0; + foreach (var (name, distribution) in searchSpace) + { + if (!ReferenceEquals(_lastKeyDistributions![i], distribution) || !string.Equals(_lastKeyNames[i], name, StringComparison.Ordinal)) + { + same = false; + break; + } + i++; + } + if (same) + return _lastKeyQmcId; + } + // Sort by keys to make the order static. - var searchSpaceStr = new OrderedDict(); + var searchSpaceStr = new OrderedDict(searchSpace.Count); foreach (var name in searchSpace.Keys.OrderBy(k => k, StringComparer.Ordinal)) searchSpaceStr[name] = searchSpace[name].ToString(); @@ -277,12 +331,20 @@ private int FindSampleId(Study study, OrderedDict sear byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(PyJson.Dumps(qmcVars))); string keyQmcId = "qmc:" + Convert.ToHexString(hash).ToLowerInvariant(); - var studyAttrs = study.Storage.GetStudySystemAttrs(study.StudyId); - int sampleId = (studyAttrs.TryGetValue(keyQmcId, out var v) && v is not null - ? Convert.ToInt32(v) - : -1) + 1; - study.Storage.SetStudySystemAttr(study.StudyId, keyQmcId, sampleId); - return sampleId; + // Remember the space this key belongs to, by its names and the distribution objects themselves. + var names = new string[searchSpace.Count]; + var distributions = new BaseDistribution[searchSpace.Count]; + int position = 0; + foreach (var (name, distribution) in searchSpace) + { + names[position] = name; + distributions[position] = distribution; + position++; + } + _lastKeyNames = names; + _lastKeyDistributions = distributions; + _lastKeyQmcId = keyQmcId; + return keyQmcId; } /// Warns that independently seeded scrambled sequences lose their shared low-discrepancy structure. diff --git a/test/OptunaSharp.Tests.Interop/QmcSamplerParityTests.cs b/test/OptunaSharp.Tests.Interop/QmcSamplerParityTests.cs index b6d192a..1ec1bf2 100644 --- a/test/OptunaSharp.Tests.Interop/QmcSamplerParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/QmcSamplerParityTests.cs @@ -225,4 +225,51 @@ public void ScrambledHalton_ManyDims_FullState_IsBitIdentical() " return v\n" + "study.optimize(_obj, n_trials=20)\n"); } + + /// + /// Plan 14 gate: QMC now computes its qmc:<sha256> sample-id key once per search space and reuses it while + /// the same distribution objects come back. A 40-parameter study (float, log-float, int and categorical) over 300 + /// trials, for each engine and scrambling, must keep the whole state, the counter in the study system attrs + /// included, bit-identical to live Optuna. + /// + /// The engine: sobol or halton. + /// Whether the engine is scrambled (the seed then enters the key). + [TestMethod] + [TestCategory("RequiresPython")] + [DataRow("sobol", false)] + [DataRow("halton", false)] + [DataRow("sobol", true)] + [DataRow("halton", true)] + public void LongRun_40Parameters_300Trials_FullState_IsBitIdentical(string qmcType, bool scramble) + { + PythonSession.EnsureOrInconclusive(); + + var csStudy = Optuna.CreateStudy( + storage: new InMemoryStorage(), studyName: "qmc", + sampler: new QMCSampler(qmcType: qmcType, scramble: scramble, seed: 11)); + csStudy.Optimize(t => + { + double v = 0; + for (int i = 0; i < 37; i++) + v += t.SuggestFloat($"x{i}", -1, 1); + v += Math.Log(t.SuggestFloat("lx", 1e-3, 1e3, log: true)); + v += t.SuggestInt("n", -5, 5); + v += t.SuggestCategorical("c", new[] { "a", "b", "cc" }).Length; + return v; + }, nTrials: 300); + + AssertFullParity(csStudy, + "import math\n" + + "study = optuna.create_study(study_name='qmc', " + + $"sampler=optuna.samplers.QMCSampler(qmc_type='{qmcType}', scramble={(scramble ? "True" : "False")}, seed=11))\n" + + "def _obj(t):\n" + + " v = 0.0\n" + + " for i in range(37):\n" + + " v += t.suggest_float(f'x{i}', -1, 1)\n" + + " v += math.log(t.suggest_float('lx', 1e-3, 1e3, log=True))\n" + + " v += t.suggest_int('n', -5, 5)\n" + + " v += len(t.suggest_categorical('c', ['a', 'b', 'cc']))\n" + + " return v\n" + + "study.optimize(_obj, n_trials=300)\n"); + } } diff --git a/test/OptunaSharp.Tests/QmcSamplerTests.cs b/test/OptunaSharp.Tests/QmcSamplerTests.cs index fe713ef..84f2e6d 100644 --- a/test/OptunaSharp.Tests/QmcSamplerTests.cs +++ b/test/OptunaSharp.Tests/QmcSamplerTests.cs @@ -344,4 +344,38 @@ List Run() Run().Should().Equal(Run()); } + + /// + /// Plan 14: the sample-id attribute key is computed once per search space and reused while the same names come with + /// the very same distribution objects. A memoized key must always equal the key a fresh sampler computes: for the + /// same objects again (reused), for equal-by-value objects that print differently (Python equality makes + /// True == 1, but str() differs, and so must the key), for a reordered space, and back again. + /// + /// Whether the engine is scrambled (the seed then enters the key). + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public void QmcIdKey_Memoized_EqualsAFreshComputation(bool scramble) + { + var x = new FloatDistribution(-5, 5); + var flags = new CategoricalDistribution(new CategoricalChoice[] { true, false }); + var ints = new CategoricalDistribution(new CategoricalChoice[] { 1L, 0L }); + flags.Equals(ints).Should().BeTrue("Python equality: True == 1 and False == 0"); + var spaceFlags = new OrderedDict { ["x"] = x, ["c"] = flags }; + var spaceInts = new OrderedDict { ["x"] = x, ["c"] = ints }; + var reordered = new OrderedDict { ["c"] = flags, ["x"] = x }; + + string Fresh(OrderedDict space) => new QMCSampler(scramble: scramble, seed: 3).QmcIdKey(space); + + var memo = new QMCSampler(scramble: scramble, seed: 3); + string first = memo.QmcIdKey(spaceFlags); + first.Should().Be(Fresh(spaceFlags)); + memo.QmcIdKey(spaceFlags).Should().Be(first, "the same objects reuse the key"); + string ints2 = memo.QmcIdKey(spaceInts); + ints2.Should().Be(Fresh(spaceInts)); + ints2.Should().NotBe(first, "equal choices that print differently give a different key, as in Optuna"); + memo.QmcIdKey(reordered).Should().Be(Fresh(reordered)); + memo.QmcIdKey(spaceFlags).Should().Be(first); + first.Should().MatchRegex("^qmc:[0-9a-f]{64}$"); + } } From 45fadd19f317074a912bf7fb10a30a5c173bc6cc Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 14:05:41 +0200 Subject: [PATCH 021/145] perf(journal): keep sparse line offsets in the file backend (plan 14, step D; Rustuna 82b13d8); 160 MB -> 0.1 MB on a 4.3M-line journal, same records JournalFileBackend kept the byte offset of every log line it had read (Python's _log_number_offset dict), so a sync can seek to the next unread line. A 100,000-trial, 40-parameter study writes about 4.3 million lines: the dictionary alone held 160 MB for the life of the storage. Now it keeps what a read actually needs (Rustuna's scheme): the furthest line start any read has established (a sync asks for exactly the next unread line, so it still seeks straight there) and one checkpoint every OffsetCheckpointInterval = 4,096 lines. A read for an earlier line (another storage over a shared backend that is behind, a replay from the start) seeks to the nearest checkpoint at or before it and skips forward at most 4,095 lines, exactly as Python skips the lines before log_number_from when it reads from byte 0. The records each read yields, their numbering and the size snapshot are unchanged: - A line's successor start is recorded only after the line decodes (Python records it first and deletes it on a decode error), so an undecodable record never becomes a place a later read starts after; lines before the requested range are skipped undecoded and recorded, as Python records them. - Readers sharing a backend record the same offsets (the file only grows): the furthest start only moves forward, a checkpoint is appended by whichever reader passes it first, and a reader can only start past a checkpoint some read passed, so checkpoints stay in order. All bookkeeping is under the existing _offsetLock. Measured (journalmem probe, scratchpad p14/journalmem, built against the b3baef9 snapshot and this tree; a synthetic 4.3M-line, 1.09 GB journal of set_trial_param-shaped records): memory the backend retains after a full read: 160.3 MB -> 0.1 MB full read: 5.48-7.36 s -> 5.22-5.26 s; next sync (seek to the end): 0.28-0.35 ms -> 0.34-0.36 ms Parity coverage: - New unit JournalOffsetCheckpointTests: reads from 13 positions (both sides of each checkpoint, the end and past it) equal the matching slice of a full read on a cold and on a warm backend, and the warm backend kept 3 checkpoints for 8,692 lines; a storage-like reader that stopped half way resumes exactly after the file grows and another reader reaches the end; 12 concurrent readers from scattered positions on one backend all read exactly; an undecodable record followed by another fails every read that reaches it (from before it, from it) instead of being skipped; a partial last line is left for the next read. - New live gate JournalStateParityTests.LargeJournal_PythonWrites_CSharpReplaysAndContinues_SharedBackendCatchesUp_ FullState: Python writes 250 trials x 40 parameters (more than two checkpoint intervals); a C# storage replays Python's full state; two C# storages share one backend while one writes 30 more trials, so the other's next sync starts behind the furthest offset; the lagging storage, a cold C# storage and Python's fresh replay all give the identical full state (floats and datetimes as instants). - Journal suites: unit 95/95; live 30/30 (5 Redis cases skip without a Redis server; the Redis backend is untouched). --- .../Storages/Journal/JournalFileBackend.cs | 118 ++++++++--- .../JournalStateParityTests.cs | 55 +++++ .../Storages/JournalOffsetCheckpointTests.cs | 197 ++++++++++++++++++ 3 files changed, 345 insertions(+), 25 deletions(-) create mode 100644 test/OptunaSharp.Tests/Storages/JournalOffsetCheckpointTests.cs diff --git a/src/OptunaSharp.Core/Storages/Journal/JournalFileBackend.cs b/src/OptunaSharp.Core/Storages/Journal/JournalFileBackend.cs index 097666a..8bd75f3 100644 --- a/src/OptunaSharp.Core/Storages/Journal/JournalFileBackend.cs +++ b/src/OptunaSharp.Core/Storages/Journal/JournalFileBackend.cs @@ -29,14 +29,37 @@ public class JournalFileBackend : BaseJournalBackend private readonly string _filePath; private readonly BaseJournalFileLock _lock; - // Byte offset of every log number seen so far, so a sync can seek instead of re-reading the journal. Python's - // dict is safe to share between readers under the GIL; this Dictionary is not, and one backend instance can be - // read by several JournalStorage instances at once (a singleton backend under scoped or transient storages). - // Every access therefore takes _offsetLock, and a reader never looks up an entry another reader may still be - // writing (see ReadLogs). - private readonly Dictionary _logNumberOffset = new() { [0] = 0 }; + /// + /// How many log lines apart the kept start offsets lie (plan 14; Rustuna 82b13d8). Python keeps one offset per line + /// (_log_number_offset); a 100,000-trial, 40-parameter journal has about 4.5 million lines, 120-250 MB of + /// dictionary. Here only the furthest offset reached and one offset per this many lines are kept. + /// + internal const int OffsetCheckpointInterval = 4096; + + // Where a read can start without scanning from the file's beginning. Python's dict is safe to share between + // readers under the GIL; these fields are not, and one backend instance can be read by several JournalStorage + // instances at once (a singleton backend under scoped or transient storages). Every access therefore takes + // _offsetLock, and each reader carries its own byte position (see ReadLogs). + // The furthest log number whose start offset a read has established, and that offset. A sync asks for exactly the + // next unread log, which is this one, so it seeks straight there as Python's per-line dictionary lets it. + private int _latestLogNumber; + private long _latestLogOffset; + // The start offset of log number k * OffsetCheckpointInterval at index k (log 0 starts at byte 0). A read for an + // earlier log number (a new storage over a shared backend, a replay from the start) seeks to the nearest checkpoint + // at or before it and skips forward, never more than OffsetCheckpointInterval - 1 lines. + private readonly List _offsetCheckpoints = new() { 0 }; private readonly object _offsetLock = new(); + /// How many offset checkpoints are kept, for the tests that check the bookkeeping stays sparse. + internal int OffsetCheckpointCount + { + get + { + lock (_offsetLock) + return _offsetCheckpoints.Count; + } + } + /// /// The lock this backend acquires around every append: the caller's, or the /// . Exposed so tests can check which one a constructor chose. @@ -92,21 +115,20 @@ public JournalFileBackend(string filePath, BaseJournalFileLock? lockObj = null) using var f = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); // Maintain remaining_log_size to allow writing by another process while reading the log. long remainingLogSize = new FileInfo(_filePath).Length; - int logNumberStart = 0; // Byte offset of the line being read. Python reads `_log_number_offset[log_number]` for it, but that entry - // can be missing here while another reader of this backend is still inserting it, so the position is - // carried by this enumeration instead; it equals the cached offset whenever both exist. - long position = 0; - bool known; - long offset; + // can be missing here while another reader of this backend is still recording it, so the position is + // carried by this enumeration instead; it equals the recorded offset whenever both exist. + // Python seeks to `log_number_from` when it has that line's offset and reads from byte 0 otherwise; this starts + // at the nearest known line start at or before it instead (plan 14). Either way the lines before + // `log_number_from` are skipped exactly as below, so the records yielded and the size accounting are the same. + int logNumberStart; + long position; lock (_offsetLock) - known = _logNumberOffset.TryGetValue(logNumberFrom, out offset); - if (known) + (logNumberStart, position) = ReadStart(logNumberFrom); + if (position > 0) { - f.Seek(offset, SeekOrigin.Begin); - logNumberStart = logNumberFrom; - remainingLogSize -= offset; - position = offset; + f.Seek(position, SeekOrigin.Begin); + remainingLogSize -= position; } Exception? lastDecodeError = null; @@ -127,12 +149,14 @@ public JournalFileBackend(string filePath, BaseJournalFileLock? lockObj = null) logNumber++; continue; } - // First writer wins, as Python's `if log_number + 1 not in ...`; every reader computes the same value. - lock (_offsetLock) - _logNumberOffset.TryAdd(logNumber + 1, position + byteLen); - position += byteLen; + long nextOffset = position + byteLen; + position = nextOffset; if (logNumber < logNumberFrom) { + // A line before the requested range is skipped undecoded, as Python skips it, and its successor's start + // is recorded as Python records it. + lock (_offsetLock) + RecordLineStart(logNumber + 1, nextOffset); logNumber++; continue; } @@ -146,17 +170,61 @@ public JournalFileBackend(string filePath, BaseJournalFileLock? lockObj = null) } catch (ValueError err) { + // A record that does not decode must not become a seek target, or the next sync would skip it: its + // successor's start is recorded only after it decodes (Python records it first and deletes it here). lastDecodeError = err; - // A record that does not decode must not become a seek target, or the next sync would skip it. - lock (_offsetLock) - _logNumberOffset.Remove(logNumber + 1); } if (parsed is not null) + { + lock (_offsetLock) + RecordLineStart(logNumber + 1, nextOffset); yield return parsed; + } logNumber++; } } + /// The known line start a read for begins at; the caller holds _offsetLock. + /// The first log number the read must yield. + /// + /// The log number of the line the read starts at and that line's byte offset: the furthest line start established + /// when is at or past it, otherwise the checkpoint at or before it (log 0 at byte 0 + /// when nothing nearer is known). + /// + private (int LogNumber, long Offset) ReadStart(int logNumberFrom) + { + if (logNumberFrom >= _latestLogNumber) + return (_latestLogNumber, _latestLogOffset); + int index = logNumberFrom / OffsetCheckpointInterval; + // Every checkpoint below the furthest line start has been recorded: the read that established it passed each. + return index < _offsetCheckpoints.Count + ? (index * OffsetCheckpointInterval, _offsetCheckpoints[index]) + : (0, 0L); + } + + /// + /// Records that log number starts at byte , once the line + /// before it has been read whole and decoded; the caller holds _offsetLock. + /// + /// The log number whose line starts at . + /// The byte offset of that line's first byte. + /// + /// Readers sharing the backend record the same offsets for the same lines (the file only grows), so the order + /// they record in does not matter: the furthest start only moves forward, and a checkpoint is appended by + /// whichever reader passes it first. A reader that starts past a checkpoint starts at a line some reader reached by + /// reading through every earlier checkpoint, so checkpoints are appended in order. + /// + private void RecordLineStart(int logNumber, long offset) + { + if (logNumber > _latestLogNumber) + { + _latestLogNumber = logNumber; + _latestLogOffset = offset; + } + if (logNumber % OffsetCheckpointInterval == 0 && logNumber / OffsetCheckpointInterval == _offsetCheckpoints.Count) + _offsetCheckpoints.Add(offset); + } + /// /// Streams raw lines (including the \n terminator, like Python's binary-file line /// iteration) so byte lengths match the offsets Python would compute. diff --git a/test/OptunaSharp.Tests.Interop/JournalStateParityTests.cs b/test/OptunaSharp.Tests.Interop/JournalStateParityTests.cs index 04456ee..d1be4ba 100644 --- a/test/OptunaSharp.Tests.Interop/JournalStateParityTests.cs +++ b/test/OptunaSharp.Tests.Interop/JournalStateParityTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using AwesomeAssertions; using OptunaSharp.Samplers; using OptunaSharp.Storages.Journal; using Python.Runtime; @@ -195,6 +196,60 @@ double Objective(Trial t) StudySnapshot.AssertParity(pyState, csState, DatetimeComparison.Instant); } + // ------------------------------------------------------------------ large journals (plan 14 offset checkpoints) + + /// + /// Plan 14 gate: the file backend now keeps one offset per JournalFileBackend.OffsetCheckpointInterval lines + /// (4,096) plus the furthest one, where Python keeps one per line. Python writes a 40-parameter study of 250 trials + /// (over 10,000 lines, more than two checkpoint intervals); a C# storage replays it to Python's full state. Two C# + /// storages then share one backend: one stays synced at Python's end while the other writes 30 more trials, so the + /// lagging one's next sync starts behind the furthest offset (at a checkpoint, skipping forward). Every replay — + /// the lagging storage, a cold C# storage and Python's — must give the identical full state, floats and datetimes + /// included. + /// + [TestMethod] + [TestCategory("RequiresPython")] + public void LargeJournal_PythonWrites_CSharpReplaysAndContinues_SharedBackendCatchesUp_FullState() + { + PyExec( + $"storage = {PyJournalStorageExpr(_path)}\n" + + "study = optuna.create_study(study_name='big', storage=storage, " + + "sampler=optuna.samplers.RandomSampler(seed=14))\n" + + "def objective(t):\n" + + " return sum(t.suggest_float(f'x{i}', -10, 10) ** 2 for i in range(40))\n" + + "study.optimize(objective, n_trials=250)\n"); + var pyWritten = PyStudyState("study"); + File.ReadAllLines(_path).Length.Should().BeGreaterThan(2 * JournalFileBackend.OffsetCheckpointInterval); + + var shared = new JournalFileBackend(_path, lockObj: new JournalFileOpenLock(_path)); + var writer = Optuna.LoadStudy("big", new JournalStorage(shared), sampler: new RandomSampler(seed: 15)); + var lagging = Optuna.LoadStudy("big", new JournalStorage(shared), sampler: new RandomSampler(0)); + StudySnapshot.AssertParity(pyWritten, StudySnapshot.Capture(writer), DatetimeComparison.Instant); + StudySnapshot.AssertParity(pyWritten, StudySnapshot.Capture(lagging), DatetimeComparison.Instant); + + writer.Optimize(t => + { + double sum = 0; + for (int i = 0; i < 40; i++) + { + double v = t.SuggestFloat($"x{i}", -10, 10); + sum += v * v; + } + return sum; + }, nTrials: 30); + var written = StudySnapshot.Capture(writer); + written.Trials.Should().HaveCount(280); + + StudySnapshot.AssertParity(written, StudySnapshot.Capture(lagging), DatetimeComparison.Instant); + StudySnapshot.AssertParity(written, + StudySnapshot.Capture(Optuna.LoadStudy("big", NewCSharpStorage(), sampler: new RandomSampler(0))), + DatetimeComparison.Instant); + PyExec( + $"fresh_storage = {PyJournalStorageExpr(_path)}\n" + + "fresh = optuna.load_study(study_name='big', storage=fresh_storage)\n"); + StudySnapshot.AssertParity(written, PyStudyState("fresh"), DatetimeComparison.Instant); + } + // ------------------------------------------------------------------ raw line-format parity [TestMethod] diff --git a/test/OptunaSharp.Tests/Storages/JournalOffsetCheckpointTests.cs b/test/OptunaSharp.Tests/Storages/JournalOffsetCheckpointTests.cs new file mode 100644 index 0000000..d9fe911 --- /dev/null +++ b/test/OptunaSharp.Tests/Storages/JournalOffsetCheckpointTests.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using AwesomeAssertions; +using OptunaSharp.Collections; +using OptunaSharp.Internal; +using OptunaSharp.Storages.Journal; + +namespace OptunaSharp.Tests.Storages; + +/// +/// Plan 14: keeps the start offset of the furthest line read and one checkpoint every +/// lines instead of Python's offset per line. A read must +/// still yield exactly the records of a full read from the file's start, from any position, on a backend that has read +/// nothing, everything, or part of the file, while other readers share it; and an undecodable line must still never +/// become a place a later read starts after. +/// +[TestClass] +public class JournalOffsetCheckpointTests +{ + private string _path = null!; + + /// Creates a fresh journal path in the temporary folder. + [TestInitialize] + public void SetUp() + => _path = Path.Combine(Path.GetTempPath(), $"optunasharp-offsets-{Guid.NewGuid():N}.log"); + + /// Deletes the journal and its lock files. + [TestCleanup] + public void TearDown() + { + foreach (var f in Directory.GetFiles(Path.GetDirectoryName(_path)!, Path.GetFileName(_path) + "*")) + { + try + { + File.Delete(f); + } + catch (IOException) + { + // Best effort: a lock file another test process still holds is cleaned up by the next run. + } + } + } + + /// + /// Appends records of varied lengths (so line starts are not a regular grid), numbered from + /// , in batches like a study's writes. + /// + /// The backend to append through. + /// The number of the first record. + /// How many records to append. + private static void Append(JournalFileBackend backend, int first, int count) + { + var batch = new List>(); + for (int i = first; i < first + count; i++) + { + batch.Add(new OrderedDict + { + ["op_code"] = 5L, + ["i"] = (long)i, + ["pad"] = new string('x', (i * 7919) % 97), + ["v"] = i * 0.1, + }); + if (batch.Count == 37) + { + backend.AppendLogs(batch); + batch.Clear(); + } + } + if (batch.Count > 0) + backend.AppendLogs(batch); + } + + /// Every record a read yields, as JSON text, to compare reads by value. + /// The backend to read through. + /// The first log number to read. + /// The records' JSON, in order. + private static List Read(JournalFileBackend backend, int from) + => backend.ReadLogs(from).Select(log => PyJson.Dumps(log)).ToList(); + + /// + /// A read from any position equals the matching slice of a full read, on a backend that has read nothing (cold) and + /// on one that has read the whole file (warm); and the warm backend kept only the sparse offsets. + /// + [TestMethod] + public void ReadLogs_FromAnyPosition_EqualsTheSliceOfAFullRead_ColdAndWarm() + { + int interval = JournalFileBackend.OffsetCheckpointInterval; + int total = 2 * interval + 500; + var writer = new JournalFileBackend(_path, new JournalFileOpenLock(_path)); + Append(writer, 0, total); + List reference = Read(new JournalFileBackend(_path, new JournalFileOpenLock(_path)), 0); + reference.Should().HaveCount(total); + + var warm = new JournalFileBackend(_path, new JournalFileOpenLock(_path)); + Read(warm, 0).Should().Equal(reference); + warm.OffsetCheckpointCount.Should().Be(3, "checkpoints at 0, 4096 and 8192 only"); + + int[] froms = { 0, 1, 2, interval - 1, interval, interval + 1, 5000, 2 * interval - 1, 2 * interval, 2 * interval + 1, total - 1, total, total + 5 }; + foreach (int from in froms) + { + List expected = from >= total ? new List() : reference.GetRange(from, total - from); + Read(warm, from).Should().Equal(expected, $"warm read from {from}"); + Read(new JournalFileBackend(_path, new JournalFileOpenLock(_path)), from).Should().Equal(expected, $"cold read from {from}"); + } + } + + /// + /// Two storages sharing one backend: one read stops half way, the file grows, another read reaches the new end, and a + /// read resuming from the first one's position (behind the furthest) starts at a checkpoint and yields exactly the + /// records after it. + /// + [TestMethod] + public void SharedBackend_AReaderBehindTheFurthest_ResumesExactly() + { + int interval = JournalFileBackend.OffsetCheckpointInterval; + var backend = new JournalFileBackend(_path, new JournalFileOpenLock(_path)); + Append(backend, 0, interval + 2000); + // A reader that stops after 5,000 records, as a sync of a storage that was behind would. + var firstPart = backend.ReadLogs(0).Take(5000).Select(log => PyJson.Dumps(log)).ToList(); + firstPart.Should().HaveCount(5000); + + Append(backend, interval + 2000, interval + 300); + int total = 2 * interval + 2300; + List reference = Read(new JournalFileBackend(_path, new JournalFileOpenLock(_path)), 0); + reference.Should().HaveCount(total); + Read(backend, 0).Should().Equal(reference, "a full read through the shared backend"); + + Read(backend, 5000).Should().Equal(reference.GetRange(5000, total - 5000), "the behind reader resumes at its own position"); + firstPart.Should().Equal(reference.GetRange(0, 5000)); + } + + /// + /// Concurrent readers of one backend, from scattered positions, while the file is static: every read equals its + /// slice of the reference, and the bookkeeping stays consistent (one checkpoint per interval). + /// + [TestMethod] + public void ConcurrentReaders_FromScatteredPositions_AllReadExactly() + { + int interval = JournalFileBackend.OffsetCheckpointInterval; + int total = 3 * interval + 77; + Append(new JournalFileBackend(_path, new JournalFileOpenLock(_path)), 0, total); + List reference = Read(new JournalFileBackend(_path, new JournalFileOpenLock(_path)), 0); + + var shared = new JournalFileBackend(_path, new JournalFileOpenLock(_path)); + var froms = new[] { 0, 9000, 4096, 100, 12000, 8191, 3, 6000, 12364, 4097, 2, 11111 }; + Parallel.ForEach(froms, new ParallelOptions { MaxDegreeOfParallelism = 6 }, from => + { + List expected = from >= total ? new List() : reference.GetRange(from, total - from); + Read(shared, from).Should().Equal(expected, $"read from {from}"); + }); + shared.OffsetCheckpointCount.Should().Be(4, "checkpoints at 0, 4096, 8192 and 12288"); + } + + /// + /// A record that does not decode, followed by another record, fails the read with Python's error; its successor + /// never becomes a start position, so a later read from it, or from anywhere before it, fails the same way instead of + /// skipping it. + /// + [TestMethod] + public void UndecodableRecord_FollowedByAnother_FailsEveryReadThatReachesIt() + { + int interval = JournalFileBackend.OffsetCheckpointInterval; + var backend = new JournalFileBackend(_path, new JournalFileOpenLock(_path)); + Append(backend, 0, interval + 10); + File.AppendAllText(_path, "{\"op_code\": 5, broken\n", Encoding.UTF8); + Append(backend, interval + 11, 20); + + Action full = () => Read(backend, 0); + full.Should().Throw(); + Action fromBefore = () => Read(backend, interval + 5); + fromBefore.Should().Throw("the read must reach the bad record again, not start after it"); + Action fromIt = () => Read(backend, interval + 10); + fromIt.Should().Throw(); + // Stops after one record, before the bad one: the record before it still reads. + backend.ReadLogs(interval + 9).Take(1).Count().Should().Be(1); + } + + /// + /// A last line still being written (no newline yet) is left for the next read, as in Python; once completed, the next + /// read from the furthest position yields it. + /// + [TestMethod] + public void PartialLastLine_IsLeftForTheNextRead() + { + var backend = new JournalFileBackend(_path, new JournalFileOpenLock(_path)); + Append(backend, 0, 10); + File.AppendAllText(_path, "{\"op_code\": 5, \"i\": 10", Encoding.UTF8); + Read(backend, 0).Should().HaveCount(10); + File.AppendAllText(_path, "}\n", Encoding.UTF8); + var rest = Read(backend, 10); + rest.Should().ContainSingle().Which.Should().Contain("\"i\": 10"); + } +} From 2dc35cfd31b0fe67457f266b2e4a7a7c811d36b8 Mon Sep 17 00:00:00 2001 From: Eli Belash Date: Mon, 9 Nov 2026 14:13:09 +0200 Subject: [PATCH 022/145] perf(sqlite): refresh running trials with UNION ALL instead of OR (plan 14, step E; Rustuna d46ace5); 8,000 trials with a stale running trial 13.0 -> 9.6 s, same rows CachedStorage refreshes a study by asking the RDB storage for trial_id > watermark (the last finished trial) plus the unfinished trials it tracks at or below it: Optuna's query is `... WHERE study_id = ? AND (trial_id > ? OR trial_id IN (...)) ORDER BY trial_id`. That second part is non-empty only while a trial below the watermark is still RUNNING: a parallel worker's, or one a crashed worker left behind (then on every refresh for the rest of the study). SQLite answers the OR by walking every trial of the study in ix_trials_study_id: 2.6 ms per refresh at 20,000 trials (plan 14 research, scratch p14/sqlplan), so such a study's per-trial cost grew with its whole history. The SQLite RDBStorage now sends the two halves joined by UNION ALL, each indexable through the primary key (64 us there): `