From 58ff72098cabfd08d812c90f056c036c3a96883a Mon Sep 17 00:00:00 2001 From: paulnord Date: Sat, 5 Sep 2026 17:57:16 -0400 Subject: [PATCH 01/21] Improve curve-fit precision, spreadsheet reports, and results layout --- .../resources/tools/tools.properties | 51 +++- .../tools/CurveFitReport.java | 124 +++++++++ .../opensourcephysics/tools/DataToolTab.java | 16 +- .../tools/DatasetCurveFitter.java | 244 ++++++++++++++---- test/README.md | 21 ++ .../tools/CurveFitDataToolLayoutTest.java | 63 +++++ .../tools/CurveFitPopupTest.java | 87 +++++++ .../tools/CurveFitPrecisionTest.java | 59 +++++ .../tools/CurveFitReportTest.java | 101 ++++++++ 9 files changed, 716 insertions(+), 50 deletions(-) create mode 100644 src/org/opensourcephysics/tools/CurveFitReport.java create mode 100644 test/README.md create mode 100644 test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java create mode 100644 test/org/opensourcephysics/tools/CurveFitPopupTest.java create mode 100644 test/org/opensourcephysics/tools/CurveFitPrecisionTest.java create mode 100644 test/org/opensourcephysics/tools/CurveFitReportTest.java diff --git a/src/org/opensourcephysics/resources/tools/tools.properties b/src/org/opensourcephysics/resources/tools/tools.properties index 2540c9c7..94d28661 100644 --- a/src/org/opensourcephysics/resources/tools/tools.properties +++ b/src/org/opensourcephysics/resources/tools/tools.properties @@ -905,4 +905,53 @@ LibraryResource.Type.Data.Description=Data file LibraryResource.Type.URL.Description=URL hyperlink LibraryBrowser.Dialog.NoResources.File=The file LibraryTreePanel.Dialog.Open.Message=Do you want to open -LibraryTreePanel.Dialog.Open.Title=Open Resource? \ No newline at end of file +LibraryTreePanel.Dialog.Open.Title=Open Resource? + +# Curve fit clipboard report +DatasetCurveFitter.Button.CopyFitReport=Copy Fit Report +DatasetCurveFitter.Button.CopyFitReport.Tooltip=Copy a spreadsheet-ready regression summary, ANOVA, and coefficients +DatasetCurveFitter.Report.Title=Curve fit report +DatasetCurveFitter.Report.Data=Data +DatasetCurveFitter.Report.Model=Model +DatasetCurveFitter.Report.Equation=Equation +DatasetCurveFitter.Report.Mode=Mode +DatasetCurveFitter.Report.Auto=Automatic fit +DatasetCurveFitter.Report.Manual=Manual parameters +DatasetCurveFitter.Report.Parameter=Parameter +DatasetCurveFitter.Report.Value=Value (full precision) +DatasetCurveFitter.Report.Uncertainty=Uncertainty (full precision) +DatasetCurveFitter.Report.Rounded=Value +/- uncertainty (rounded) +DatasetCurveFitter.Report.Fixed=Fixed +DatasetCurveFitter.Report.Yes=Yes +DatasetCurveFitter.Report.No=No +DatasetCurveFitter.Report.NA=N/A +DatasetCurveFitter.Report.Points=Data points (n) +DatasetCurveFitter.Report.Free=Free parameters (p) +DatasetCurveFitter.Report.DF=Residual degrees of freedom (n - p) +DatasetCurveFitter.Report.SSE=Residual sum of squares (SSE) +DatasetCurveFitter.Report.RMS=RMS deviation (sqrt(SSE/n)) +DatasetCurveFitter.Report.R2=R-squared (1 - SSE/SST) +DatasetCurveFitter.Report.SE=Residual standard error (sqrt(SSE/(n-p))) +DatasetCurveFitter.Report.Note=Residual statistics for the current data and parameters. Parameter uncertainties are the fitter's estimates. N/A means unavailable or undefined. + +DatasetCurveFitter.Button.CopyFitParameters=Copy Fit Parameters +DatasetCurveFitter.Button.CopyFitParameters.Tooltip=Copy full-precision values, uncertainties, and rounded reporting values +DatasetCurveFitter.Statistics.Points=Points +DatasetCurveFitter.Statistics.Free=Free parameters +DatasetCurveFitter.Statistics.DF=Degrees of freedom +DatasetCurveFitter.Statistics.R2=R-squared +DatasetCurveFitter.Statistics.SSE=SSE +DatasetCurveFitter.Statistics.SE=Residual SE + +DatasetCurveFitter.Report.Summary=SUMMARY OUTPUT +DatasetCurveFitter.Report.RegressionStatistics=Regression Statistics +DatasetCurveFitter.Report.MultipleR=Multiple R +DatasetCurveFitter.Report.RSquare=R Square +DatasetCurveFitter.Report.AdjustedRSquare=Adjusted R Square +DatasetCurveFitter.Report.StandardError=Standard Error +DatasetCurveFitter.Report.Observations=Observations +DatasetCurveFitter.Report.ANOVA=ANOVA +DatasetCurveFitter.Report.Regression=Regression +DatasetCurveFitter.Report.Residual=Residual +DatasetCurveFitter.Report.Total=Total +DatasetCurveFitter.Report.Coefficients=Coefficients diff --git a/src/org/opensourcephysics/tools/CurveFitReport.java b/src/org/opensourcephysics/tools/CurveFitReport.java new file mode 100644 index 00000000..c10dbc03 --- /dev/null +++ b/src/org/opensourcephysics/tools/CurveFitReport.java @@ -0,0 +1,124 @@ +package org.opensourcephysics.tools; + +import org.opensourcephysics.display.Dataset; +import org.opensourcephysics.display.OSPRuntime; + +/** Tab-separated snapshots of curve fits for spreadsheets and reports. */ +final class CurveFitReport { + private CurveFitReport() {} + static final String[] STATISTIC_KEYS = {"Points", "Free", "DF", "SSE", "RMS", "R2", "SE"}; + + static String create(KnownFunction fit, Dataset data, boolean[] fixed, + double[] uncertainties, boolean autofit, boolean includeStatistics) { + StringBuilder out = new StringBuilder(); + if (includeStatistics) { + String xName = data == null ? "x" : data.getXColumnName(); + String yName = data == null ? "y" : data.getYColumnName(); + String dataName = data == null ? "" : data.getName(); + if (dataName == null || dataName.trim().isEmpty()) dataName = yName + " vs " + xName; + row(out, label("Summary"), dataName); + row(out, label("Model"), fit.getName(), label("Equation"), + yName + " = " + fit.getExpression(xName), label(autofit ? "Auto" : "Manual")); + row(out); + double[] stats = statistics(fit, data, fixed, autofit); + double n = stats[0], p = stats[1], df = stats[2], sse = stats[3], r2 = stats[5], sst = stats[7]; + row(out, label("RegressionStatistics")); + row(out, label("MultipleR"), number(r2 >= 0 ? Math.sqrt(r2) : Double.NaN)); + row(out, label("RSquare"), number(r2)); + row(out, label("AdjustedRSquare"), number(autofit && df > 0 && n > 1 && sst > 0 + ? 1 - (sse / df) / (sst / (n - 1)) : Double.NaN)); + row(out, label("StandardError"), number(stats[6])); + row(out, label("Observations"), Integer.toString((int) n)); + row(out); + row(out, label("ANOVA")); + row(out, "", "df", "SS", "MS", "F"); + // Classical regression ANOVA applies to an automatically fitted, + // full-rank polynomial with a freely fitted intercept and no constraints. + boolean regression = autofit && fit instanceof KnownPolynomial && p == fit.getParameterCount() + && df > 0 && p > 1 && sst > 0 && sse >= 0 && sse <= sst + && hasDistinctAbscissas(data, (int) p); + double regressionDF = regression ? p - 1 : Double.NaN; + double regressionSS = regression ? sst - sse : Double.NaN; + double regressionMS = regressionSS / regressionDF; + double residualMS = df > 0 ? sse / df : Double.NaN; + row(out, label("Regression"), integer(regressionDF), number(regressionSS), number(regressionMS), + number(regression && residualMS > 0 ? regressionMS / residualMS : Double.NaN)); + row(out, label("Residual"), integer(df), number(sse), number(residualMS)); + row(out, label("Total"), n > 0 ? Integer.toString((int) n - 1) : label("NA"), number(sst)); + row(out); + } + // Exactly one numeric coefficient and uncertainty per parameter. No + // duplicate rounded column: rounding belongs to the on-screen display. + row(out, label("Parameter"), label("Coefficients"), label("StandardError"), label("Fixed")); + for (int i = 0; i < fit.getParameterCount(); i++) { + boolean isFixed = fixed != null && i < fixed.length && fixed[i]; + double sigma = autofit && !isFixed && uncertainties != null && i < uncertainties.length + ? uncertainties[i] : Double.NaN; + if (sigma < 0) sigma = Double.NaN; + row(out, fit.getParameterName(i), number(fit.getParameterValue(i)), number(sigma), label(isFixed ? "Yes" : "No")); + } + return out.toString(); + } + + private static boolean hasDistinctAbscissas(Dataset data, int required) { + java.util.HashSet distinct = new java.util.HashSet(); + for (double x : data.getValidXPoints()) { + if (!Double.isFinite(x)) return false; + distinct.add(x == 0 ? 0.0 : x); + if (distinct.size() >= required) return true; + } + return false; + } + + private static String integer(double value) { + return Double.isFinite(value) ? Integer.toString((int) value) : label("NA"); + } + + /** n, p, df, SSE, RMS, R-squared, residual standard error, SST; shared by screen and export. */ + static double[] statistics(KnownFunction fit, Dataset data, boolean[] fixed, boolean autofit) { + int freeParameters = 0; + for (int i = 0; i < fit.getParameterCount(); i++) + if (fixed == null || i >= fixed.length || !fixed[i]) freeParameters++; + double[] x = data == null ? new double[0] : data.getValidXPoints(); + double[] y = data == null ? new double[0] : data.getValidYPoints(); + int n = x.length; + double sse = 0; + double mean = 0; + double sst = 0; + for (int i = 0; i < n; i++) { + double predicted = fit.evaluate(x[i]); + double residual = y[i] - predicted; + sse += residual * residual; + double delta = y[i] - mean; + mean += delta / (i + 1); + sst += delta * (y[i] - mean); + if (!Double.isFinite(x[i]) || !Double.isFinite(y[i]) || !Double.isFinite(predicted) + || (fit instanceof UserFunction && ((UserFunction) fit).evaluatedToNaN())) + sse = Double.NaN; + } + int df = n - freeParameters; + boolean valid = n > 0 && Double.isFinite(sse); + return new double[] {n, freeParameters, autofit && n > 0 ? df : Double.NaN, + valid ? sse : Double.NaN, valid ? Math.sqrt(sse / n) : Double.NaN, + valid && Double.isFinite(sst) && sst > 0 ? 1 - sse / sst : Double.NaN, + valid && autofit && df > 0 ? Math.sqrt(sse / df) : Double.NaN, + n > 0 && Double.isFinite(sst) ? sst : Double.NaN}; + } + + private static String label(String key) { + return ToolsRes.getString("DatasetCurveFitter.Report." + key); + } + + private static String number(double value) { + return Double.isFinite(value) + ? Double.toString(value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : label("NA"); + } + + private static void row(StringBuilder out, String... cells) { + for (int i = 0; i < 5; i++) { + if (i > 0) out.append('\t'); + out.append(i >= cells.length || cells[i] == null ? "" : cells[i].replace('\t', ' ').replace('\n', ' ').replace('\r', ' ')); + } + out.append('\n'); + } +} diff --git a/src/org/opensourcephysics/tools/DataToolTab.java b/src/org/opensourcephysics/tools/DataToolTab.java index f45b4894..6db28890 100644 --- a/src/org/opensourcephysics/tools/DataToolTab.java +++ b/src/org/opensourcephysics/tools/DataToolTab.java @@ -1180,7 +1180,21 @@ public void propertyChange(PropertyChangeEvent e) { splitPanes[0].setResizeWeight(0.7); splitPanes[0].setOneTouchExpandable(true); // splitPanes[1] is plot on top, fitter on bottom - splitPanes[1] = new JSplitPane(JSplitPane.VERTICAL_SPLIT); + splitPanes[1] = new JSplitPane(JSplitPane.VERTICAL_SPLIT) { + @Override + public void doLayout() { + if (getBottomComponent() instanceof DatasetCurveFitter && getHeight() > 0) { + DatasetCurveFitter fitter = (DatasetCurveFitter) getBottomComponent(); + java.awt.Insets insets = getInsets(); + int width = getWidth() - insets.left - insets.right; + int required = fitter.prepareFitLayout(width); + setDividerLocation(Math.max(insets.top, + getHeight() - insets.bottom - getDividerSize() - required)); + } + super.doLayout(); + } + + }; splitPanes[1].setResizeWeight(1); splitPanes[1].setDividerSize(0); // splitPanes[2] is stats/props tables on top, data table on bottom diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index 5677cd12..13526e83 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -253,17 +253,21 @@ public void setAutoFit(boolean autofit) { autofitCheckBox.setSelected(autofit); if (!autofit) drawer.setUncertainties(null); + refreshFitStatistics(); } // GUI - private JButton colorButton, closeButton; + private JButton colorButton, closeButton, copyFitReportButton; private JCheckBox autofitCheckBox; private JLabel fitLabel, eqnLabel, rmsLabel; private JToolBar fitBar, eqnBar, rmsBar; private JComboBox fitDropDown; private JTextField eqnField; private NumberField rmsField; + private JPanel statisticsPanel; + private JLabel[] statisticLabels; + private static final int[] VISIBLE_STATISTICS = {0, 1, 2, 5, 3, 6}; private ParamTable paramTable; private ParamCellRenderer cellRenderer; private SpinCellEditor spinCellEditor; // uses number-crawler spinner @@ -412,6 +416,7 @@ public double fit(KnownFunction fit, boolean fromScratch) { paramTable.setEnabled(false); rmsField.setText(ToolsRes.getString("DatasetCurveFitter.RMSField.NoData")); //$NON-NLS-1$ rmsField.setForeground(Color.RED); + refreshFitStatistics(); return Double.NaN; } @@ -560,6 +565,8 @@ public double fit(KnownFunction fit, boolean fromScratch) { rmsField.setValue(rmsDev); rmsField.setToolTipText(null); } + refreshFitStatistics(); + refreshParameterLayout(); refreshStatusBar(); firePropertyChange(PROPERTY_DATASETCURVEFITTER_FIT, null, null); if (tab != null && tab.areaVisible && tab.measureFit) @@ -568,6 +575,36 @@ public double fit(KnownFunction fit, boolean fromScratch) { return rmsDev; } + /** Refreshes display-only residual statistics without running the optimizer. */ + private void refreshFitStatistics() { + if (statisticLabels == null) return; + double[] values = fit == null ? null : CurveFitReport.statistics(fit, dataset, fixedParams.get(fit), autofit); + for (int i = 0; i < statisticLabels.length; i++) { + int index = VISIBLE_STATISTICS[i]; + String key = CurveFitReport.STATISTIC_KEYS[index]; + double value = values == null ? Double.NaN : values[index]; + String text = !Double.isFinite(value) ? ToolsRes.getString("DatasetCurveFitter.Report.NA") + : index < 3 ? Integer.toString((int) value) + : String.format(java.util.Locale.ROOT, "%.5g", value).replace('.', OSPRuntime.getCurrentDecimalSeparator()); + statisticLabels[i].setText(ToolsRes.getString("DatasetCurveFitter.Statistics." + key) + ": " + text); + statisticLabels[i].setToolTipText(ToolsRes.getString("DatasetCurveFitter.Report." + key) + ": " + + (Double.isFinite(value) ? Double.toString(value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : text)); + } + } + + /** Copies a snapshot of the current fit without fitting or rounding its data. */ + private void copyFitResults(boolean includeStatistics) { + if (paramTable.isEditing() && !paramTable.getCellEditor().stopCellEditing()) + return; + if (fit == null) + return; + double[] sigma = new double[fit.getParameterCount()]; + for (int i = 0; i < sigma.length; i++) + sigma[i] = getUncertainty(i); + OSPRuntime.copy(CurveFitReport.create(fit, dataset, fixedParams.get(fit), sigma, + autofit, includeStatistics), null); + } + /** * Adds a fit function. * @@ -613,41 +650,44 @@ public double getUncertainty(int paramIndex) { } /** - * Returns two strings describing a parameter and its uncertainty. - * One for display, other with more sig figs for tooltip + * Returns a rounded parameter/uncertainty display and an unrounded tooltip. + * Formatting never changes the fitted parameter or uncertainty. * * @param value the parameter value - * @param sigma the uncertainty (may be null) - * @return the format values {decimal places, format} or null if uncert unknown or zero + * @param sigma the positive, finite uncertainty + * @param extraPlaces significant digits beyond the first uncertainty digit + * @param format retained for compatibility with existing callers + * @return {display, tooltip}, or null if the uncertainty is unknown or zero */ public String[] formatUncertainParameter(double value, double sigma, int extraPlaces, NumberFormat format) { - if (Double.isNaN(sigma) || sigma <= 0) { + return formatParameterWithUncertainty(value, sigma, extraPlaces); + } + + static String[] formatParameterWithUncertainty(double value, double sigma, int extraPlaces) { + if (Double.isNaN(sigma) || Double.isInfinite(sigma) || sigma <= 0) { return null; } - - int exp = value == 0? 0: (int) Math.floor(Math.log10(Math.abs(value))); - int expSig = sigma == 0? 0: (int) Math.floor(Math.log10(Math.abs(sigma))); - if (expSig > exp) - exp = expSig; - int shift = exp - expSig; - double multiplier = Math.pow(10, -exp); - int places = Math.max(0, shift) + extraPlaces; - - String val = String.format("%." + places + "f", value*multiplier); - String sig = String.format("%." + places + "f", sigma*multiplier); + // Find the uncertainty exponent AFTER rounding, including carries such as + // 0.0996 -> 0.10. Both displayed numbers must end at the same place. + String roundedSigma = String.format(java.util.Locale.ROOT, "%." + extraPlaces + "e", sigma); + int expSig = Integer.parseInt(roundedSigma.substring(roundedSigma.indexOf('e') + 1)); + int exp = value == 0 ? 0 : (int) Math.floor(Math.log10(Math.abs(value))); + exp = Math.max(exp, expSig); + int places = exp - expSig + extraPlaces; + double scale = Math.pow(10, exp); + String val = String.format(java.util.Locale.ROOT, "%." + places + "f", value / scale); + String sig = String.format(java.util.Locale.ROOT, "%." + places + "f", sigma / scale); String formatted = val + " \u00B1 " + sig; + if (exp != 0) + formatted = "(" + formatted + ") E" + exp; String separator = String.valueOf(OSPRuntime.getCurrentDecimalSeparator()); formatted = formatted.replace(".", separator); - if (exp != 0) - formatted = "(" + formatted +") " + String.format("E%d", exp); - - val = format.format(value); - sig = format.format(sigma); - String tooltip = val + " \u00B1 " + sig; - + // Double.toString preserves the stored double for subsequent calculations + // if a student transcribes the tooltip rather than the rounded report. + String tooltip = (Double.toString(value) + " \u00B1 " + Double.toString(sigma)).replace(".", separator); return new String[] {formatted, tooltip}; } - + /** * Gets a fit function by name. * @@ -673,10 +713,9 @@ public Map getSelectedFitParameters() { @Override public Dimension getMinimumSize() { - Dimension dim = fitBar.getPreferredSize(); - dim.height += eqnBar.getPreferredSize().height; - dim.height += rmsBar.getPreferredSize().height + 1; - return dim; + if (statisticsPanel == null) return super.getMinimumSize(); + return new Dimension(fitBar.getPreferredSize().width, + splitPane.getPreferredSize().height + statisticsPanel.getPreferredSize().height + 4); } // _______________________ protected & private methods @@ -687,7 +726,13 @@ public Dimension getMinimumSize() { */ protected void createGUI() { setLayout(new BorderLayout()); - splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT); + splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT) { + @Override + public void doLayout() { + layoutFitPane(); + super.doLayout(); + } + }; splitPane.setResizeWeight(0.7); splitPane.setDividerSize(6); // create autofit checkbox @@ -891,9 +936,12 @@ public void actionPerformed(ActionEvent e) { rmsField = new NumberField(6) { @Override public Dimension getPreferredSize() { - return fixSize(super.getPreferredSize()); + Dimension size = fixSize(super.getPreferredSize()); + size.width = Math.max(size.width, getFontMetrics(getFont()).stringWidth("-0.000E-000") + 12); + return size; } - + @Override + public Dimension getMinimumSize() { return getPreferredSize(); } }; rmsField.setEditable(false); rmsField.setEnabled(true); @@ -914,11 +962,18 @@ public void mousePressed(MouseEvent e) { }); JScrollPane scroller = new JScrollPane(paramTable) { + @Override + public Dimension getPreferredSize() { + return new Dimension(getMinimumSize().width, + paramTable.getPreferredSize().height + paramTable.getTableHeader().getPreferredSize().height + 6); + } @Override public Dimension getMinimumSize() { - Dimension dim = spinCellEditor.spinner.getPreferredSize(); - dim.width += cellRenderer.fieldFont.getSize() * 7; - return dim; + paramTable.sizeColumnsToContents(); + Dimension dim = paramTable.getMinimumSize(); + java.awt.Insets insets = getInsets(); + dim.width += insets.left + insets.right + getVerticalScrollBar().getPreferredSize().width; + return new Dimension(dim.width, spinCellEditor.spinner.getPreferredSize().height); } }; scroller.addMouseListener(new MouseAdapter() { @@ -929,7 +984,7 @@ public void mousePressed(MouseEvent e) { } }); splitPane.setRightComponent(scroller); - add(getSplitPane(), BorderLayout.CENTER); + add(getSplitPane(), BorderLayout.NORTH); // create fit builder button fitBuilderButton = DataTool.createButton(ToolsRes.getString("DatasetCurveFitter.Button.Define.Text")); //$NON-NLS-1$ fitBuilderButton.addActionListener(new ActionListener() { @@ -1010,7 +1065,21 @@ public void layoutContainer(Container target) { rmsBar.addSeparator(); rmsBar.add(rmsLabel); rmsBar.add(rmsField); + rmsBar.addSeparator(); + copyFitReportButton = new JButton(); + copyFitReportButton.addActionListener(e -> copyFitResults(true)); + rmsBar.add(copyFitReportButton); rmsPanel.add(rmsBar, BorderLayout.NORTH); + statisticsPanel = new JPanel(new java.awt.GridLayout(0, 2, 10, 3)); + statisticsPanel.setBorder(BorderFactory.createEmptyBorder(4, 4, 4, 4)); + statisticLabels = new JLabel[VISIBLE_STATISTICS.length]; + for (int i = 0; i < statisticLabels.length; i++) { + statisticLabels[i] = new JLabel(); + statisticsPanel.add(statisticLabels[i]); + } + JPanel statisticsContainer = new JPanel(new BorderLayout()); + statisticsContainer.add(statisticsPanel, BorderLayout.NORTH); + add(statisticsContainer, BorderLayout.CENTER); refreshGUI(); // refreshFitDropDown(); } @@ -1083,6 +1152,8 @@ protected void processPropertyChange(PropertyChangeEvent e) { * Refreshes the GUI. */ protected void refreshGUI() { + copyFitReportButton.setText(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport")); + copyFitReportButton.setToolTipText(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport.Tooltip")); autofitCheckBox.setText(ToolsRes.getString("Checkbox.Autofit.Label")); //$NON-NLS-1$ rmsLabel.setText(ToolsRes.getString("DatasetCurveFitter.Label.RMSDeviation")); //$NON-NLS-1$ fitBuilderButton.setText(ToolsRes.getString("DatasetCurveFitter.Button.Define.Text")); //$NON-NLS-1$ @@ -1091,12 +1162,14 @@ protected void refreshGUI() { eqnLabel.setText(ToolsRes.getString("DatasetCurveFitter.Label.Equation")); //$NON-NLS-1$ updateColorButton(); refreshFitDropDown(); + refreshFitStatistics(); } /** * Refreshes the decimal separators. */ protected void refreshDecimalSeparators() { + refreshFitStatistics(); repaint(); spinCellEditor.field.setValue(spinCellEditor.field.getValue()); } @@ -1231,18 +1304,62 @@ protected void selectFit(String name) { if (fitBuilder.isVisible()) { fitBuilder.setSelectedPanel(fit.getName()); } - paramTable.getColumnModel().getColumn(1).setMaxWidth(getMinCheckboxColumnWidth() + 10); + refreshParameterLayout(); revalidate(); } setActiveAndFit(true); } - private int getMinCheckboxColumnWidth() { - String s = ToolsRes.getString("DatasetCurveFitter.Table.Heading.FixedParam"); - Font font = paramTable.getTableHeader().getFont(); - FontMetrics fm = paramTable.getTableHeader().getFontMetrics(font); - return fm.stringWidth(s); + /** Establishes orientation and height before the parent allocates plot space. */ + int prepareFitLayout(int availableWidth) { + if (statisticsPanel == null || paramTable == null) return 0; + int width = Math.max(1, availableWidth - 4); + // Reserve room before uncertainties become available. Selection changes must + // not change the orientation or the height allocated to the plot. + int parameterWidth = paramTable.getFontMetrics(paramTable.getFont()) + .stringWidth("Parameter Fixed (-0.00000 ± 0.00000) E-000") + 64; + int controlsWidth = Math.max(fitBar.getPreferredSize().width, rmsBar.getPreferredSize().width); + boolean stacked = width < controlsWidth + parameterWidth + splitPane.getDividerSize(); + int orientation = stacked ? JSplitPane.VERTICAL_SPLIT : JSplitPane.HORIZONTAL_SPLIT; + if (splitPane.getOrientation() != orientation) { + splitPane.setOrientation(orientation); + splitPane.setResizeWeight(stacked ? 0 : 1); + } + int labelWidth = 0; + for (JLabel label : statisticLabels) { + String text = label.getText(); + int colon = text.indexOf(':'); + String reserved = (colon < 0 ? text : text.substring(0, colon + 1)) + " -0.00000E-000"; + labelWidth = Math.max(labelWidth, label.getFontMetrics(label.getFont()).stringWidth(reserved)); + } + java.awt.GridLayout layout = (java.awt.GridLayout) statisticsPanel.getLayout(); + int columns = width < 2 * labelWidth + 18 ? 1 : 2; + if (layout.getColumns() != columns) { + layout.setColumns(columns); + statisticsPanel.revalidate(); + } + int controlsHeight = splitPane.getLeftComponent().getPreferredSize().height; + int parameterHeight = splitPane.getRightComponent().getPreferredSize().height; + int infoHeight = (stacked ? controlsHeight + parameterHeight + splitPane.getDividerSize() + : Math.max(controlsHeight, parameterHeight)) + 4; + splitPane.setPreferredSize(new Dimension(availableWidth, infoHeight)); + int divider = stacked ? controlsHeight + : Math.max(controlsWidth, Math.min(splitPane.getDividerLocation(), width - parameterWidth - splitPane.getDividerSize())); + if (splitPane.getDividerLocation() != divider) splitPane.setDividerLocation(divider); + return infoHeight + statisticsPanel.getPreferredSize().height + 4; + } + + private void layoutFitPane() { + if (splitPane.getWidth() > 0) prepareFitLayout(splitPane.getWidth()); + } + + private void refreshParameterLayout() { + if (paramTable == null || cellRenderer == null || spinCellEditor == null) return; + paramTable.sizeColumnsToContents(); + paramTable.revalidate(); + if (splitPane.getOrientation() == JSplitPane.HORIZONTAL_SPLIT && splitPane.getWidth() > 0 && splitPane.getDividerLocation() > splitPane.getMaximumDividerLocation()) + splitPane.setDividerLocation(splitPane.getMaximumDividerLocation()); } protected UserFunction createClone(KnownFunction f, String name) { @@ -1902,14 +2019,32 @@ public void mousePressed(MouseEvent e) { header.addMouseListener(listener); } + /** Reserve space for the complete rendered values, including uncertainties. */ + void sizeColumnsToContents() { + if (cellRenderer == null || spinCellEditor == null || getColumnCount() != 3) return; + for (int col = 0; col < 3; col++) { + int width = getTableHeader().getFontMetrics(getTableHeader().getFont()) + .stringWidth(getColumnName(col)) + 16; + if (col == 2) width = Math.max(width, getFontMetrics(getFont()) + .stringWidth("(-0.00000 ± 0.00000) E-000") + 16); + for (int row = 0; row < getRowCount(); row++) { + Component renderer = prepareRenderer(getCellRenderer(row, col), row, col); + width = Math.max(width, renderer.getPreferredSize().width + getIntercellSpacing().width + 12); + } + javax.swing.table.TableColumn column = getColumnModel().getColumn(col); + column.setMaxWidth(col < 2 ? width : Integer.MAX_VALUE); + column.setMinWidth(width); + column.setPreferredWidth(width); + } + } + public void showPopup(MouseEvent e) { JPopupMenu popup = new JPopupMenu(); JMenuItem item = new JMenuItem(ToolsRes.getString("DatasetCurveFitter.Menuitem.CopyParameters")); //$NON-NLS-1$ - item.addActionListener((ev) -> { - selectAll(); - ActionEvent event = new ActionEvent(paramTable, ActionEvent.ACTION_PERFORMED, null); - getActionMap().get("copy").actionPerformed(event); - }); + item.addActionListener(ev -> copyFitResults(false)); + popup.add(item); + item = new JMenuItem(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport")); + item.addActionListener(ev -> copyFitResults(true)); popup.add(item); popup.addSeparator(); JCheckBoxMenuItem scientificNotationItem = new JCheckBoxMenuItem("Scientific notation"); //$NON-NLS-1$ @@ -1933,6 +2068,18 @@ public TableCellRenderer getCellRenderer(int row, int column) { return cellRenderer; } + @Override + public boolean editCellAt(int row, int column, EventObject event) { + // Reject context-menu gestures before JTable initializes the editor: + // loading the parameter spinner can otherwise disable Autofit. + if (event instanceof MouseEvent) { + MouseEvent mouse = (MouseEvent) event; + if (OSPRuntime.isPopupTrigger(mouse) || !SwingUtilities.isLeftMouseButton(mouse)) + return false; + } + return super.editCellAt(row, column, event); + } + @Override public TableCellEditor getCellEditor(int row, int column) { if (column == 1) { @@ -1965,6 +2112,7 @@ public void setFont(Font font) { } getTableHeader().setFont(font); setRowHeight(font.getSize() + 4); + refreshParameterLayout(); TableModel model = getModel(); if (model instanceof DefaultTableModel) { DefaultTableModel tm = (DefaultTableModel) model; @@ -2093,7 +2241,7 @@ public Component getTableCellRendererComponent(JTable table, Object value, boole DecimalFormat format = spinCellEditor.field.format; format.setDecimalFormatSymbols(OSPRuntime.getDecimalFormatSymbols()); double uncertainty = getUncertainty(row); - String[] uncert = formatUncertainParameter((double)value, uncertainty, 0, format); + String[] uncert = formatUncertainParameter((double)value, uncertainty, 1, format); if (Double.isNaN((Double)value)) { tooltip = ToolsRes.getString("DatasetCurveFitter.InsufficientData.ToolTip"); //$NON-NLS-1$//$NON-NLS-2$ } diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..025e6f9d --- /dev/null +++ b/test/README.md @@ -0,0 +1,21 @@ +# Curve-fit regression tests + +These standalone Java tests require a built OSP or Tracker JAR and a JDK. +They use main methods and exit unsuccessfully on assertion failures; no test +framework is required. Run from the repository root, replacing `/path/to/osp.jar` +with the built JAR: + +```sh +mkdir -p /tmp/osp-fit-tests +javac -cp /path/to/osp.jar -d /tmp/osp-fit-tests test/org/opensourcephysics/tools/*.java +for test in CurveFitPrecisionTest CurveFitReportTest CurveFitPopupTest CurveFitDataToolLayoutTest; do + java -cp /tmp/osp-fit-tests:/path/to/osp.jar org.opensourcephysics.tools.$test || exit 1 +done +``` + +The Swing tests require a graphical desktop. They create their own windows and +synthetic data. The full Data Tool test covers resizing, font sizes, parameter +visibility, compact statistics, and stable plot height across point selections. +The precision and report tests cover display rounding, full-precision values, +spreadsheet columns, and unavailable statistics. The popup test checks that +context-menu gestures preserve Autofit while left-click editing remains available. diff --git a/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java b/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java new file mode 100644 index 00000000..b4d63bf2 --- /dev/null +++ b/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java @@ -0,0 +1,63 @@ +package org.opensourcephysics.tools; +import java.awt.*; +import java.awt.event.ActionEvent; +import java.awt.image.BufferedImage; +import javax.swing.*; +import org.opensourcephysics.display.Dataset; + +/** Integration test: real Data Tool, plot/fitter split, font and window changes. */ +public class CurveFitDataToolLayoutTest { + static int passed; + static boolean success; + public static void main(String[] args)throws Exception { + SwingUtilities.invokeAndWait(()->{try{ + Dataset data=new Dataset(); data.setXYColumnNames("t","y"); + for(int i=0;i<16;i++){double t=i/30.0;data.append(t,-1980*t*t+1067*t-3.9+0.4*Math.sin(i));} + DataTool tool=new DataTool(data); tool.addNotify(); + DataToolTab tab=tool.getTab(0); tab.checkGUI(); + DatasetCurveFitter fitter=tab.getCurveFitter(); + tab.showFitterAction.actionPerformed(new ActionEvent(tab,0,fitter.getPolyFitNameOfDegree(2))); + java.lang.reflect.Field sf=DatasetCurveFitter.class.getDeclaredField("statisticsPanel");sf.setAccessible(true);JPanel stats=(JPanel)sf.get(fitter); + JScrollPane scroll=(JScrollPane)fitter.splitPane.getRightComponent();JTable table=(JTable)scroll.getViewport().getView(); + for(int level:new int[]{0,2}){ + tool.setFontLevel(level); + for(int[] size:new int[][]{{950,700},{1600,1000},{950,700}}){ + tool.setSize(size[0],size[1]); tool.validate(); + tab.splitPanes[0].setDividerLocation(0.7); + for(int i=0;i<5;i++){tool.validate();layout(tool.getContentPane());} + int stableDivider=tab.splitPanes[1].getDividerLocation(); + int stableOrientation=fitter.splitPane.getOrientation(); + for(int count:new int[]{1,2,3,8,15,16,1,16}){ + Dataset selected=new Dataset();selected.setXYColumnNames("t","y"); + for(int j=0;j=parameterBottom.y+parameterBottom.height,"parameter information above statistics"); + check(fitter.getVisibleRect().contains(statsBounds),"all statistics visible in parent fit area"); + check(stats.getHeight()==stats.getPreferredSize().height,"statistics remain compact"); + check(tab.splitPanes[1].getTopComponent().getHeight()>100,"plot retains usable height"); + check(Math.abs(fitter.getHeight()-fitter.prepareFitLayout(fitter.getWidth()))<=4,"fit area stays at required height"); + if(args.length>0){BufferedImage image=new BufferedImage(tool.getContentPane().getWidth(),tool.getContentPane().getHeight(),BufferedImage.TYPE_INT_RGB);Graphics2D g=image.createGraphics();tool.getContentPane().printAll(g);g.dispose();javax.imageio.ImageIO.write(image,"png",new java.io.File(args[0]+"/data-tool-"+size[0]+"-font"+level+".png"));} + } + } + System.out.println("Passed: "+passed); success=true; + }catch(Exception e){throw new RuntimeException(e);}finally{System.exit(success ? 0 : 1);}}); + System.exit(0); + } + static void layout(Container c){c.doLayout();for(Component child:c.getComponents())if(child instanceof Container)layout((Container)child);} + static void check(boolean b,String message){if(!b){System.err.println("FAIL: "+message);System.exit(1);}passed++;System.out.println("PASS: "+message);} +} diff --git a/test/org/opensourcephysics/tools/CurveFitPopupTest.java b/test/org/opensourcephysics/tools/CurveFitPopupTest.java new file mode 100644 index 00000000..5d8a23b4 --- /dev/null +++ b/test/org/opensourcephysics/tools/CurveFitPopupTest.java @@ -0,0 +1,87 @@ +package org.opensourcephysics.tools; + +import java.awt.Window; +import java.awt.event.InputEvent; +import java.awt.event.MouseEvent; +import javax.swing.*; +import javax.swing.table.TableCellEditor; +import org.opensourcephysics.display.Dataset; + +/** Context-menu gestures must never initialize a parameter editor. */ +public class CurveFitPopupTest { + static int passed; + public static void main(String[] args) throws Exception { + SwingUtilities.invokeAndWait(() -> { + try { + Dataset data = new Dataset(); + data.append(new double[] {0,1,2,3}, new double[] {1,3,5,7}); + DatasetCurveFitter fitter = new DatasetCurveFitter(data, new FitBuilder(null)); + fitter.fit = new KnownPolynomial(new double[] {1,2}); + fitter.setAutoFit(true); + fitter.uncertainties = new double[] {.05, .01}; + String labels = labels(fitter); + check(labels.contains("Points: 4"), "point count visible"); + check(labels.contains("Degrees of freedom: 2"), "degrees of freedom visible"); + check(labels.contains("R-squared: 1.0000"), "R-squared visible"); + check(labels.contains("SSE: 0.0000"), "SSE visible"); + check(labels.contains("Residual SE: 0.0000"), "residual standard error visible"); + boolean[] allowEditor = {false}; + DatasetCurveFitter.ParamTable table = fitter.new ParamTable(fitter.new ParamTableModel()) { + @Override public TableCellEditor getCellEditor(int row, int column) { + if (!allowEditor[0]) throw new AssertionError("Context click initialized editor"); + DefaultCellEditor editor = new DefaultCellEditor(new JTextField()); + editor.setClickCountToStart(1); + return editor; + } + }; + for (int column : new int[] {1,2}) { + reject(table, column, MouseEvent.BUTTON3, 0, true, "right-click popup"); + reject(table, column, MouseEvent.BUTTON3, 0, false, "right-click before popup release"); + reject(table, column, MouseEvent.BUTTON1, InputEvent.CTRL_DOWN_MASK, true, "Control-click popup"); + reject(table, column, MouseEvent.BUTTON1, InputEvent.CTRL_DOWN_MASK, false, "macOS Control-click"); + } + check(fitter.isAutoFit(), "Autofit preserved"); + check(fitter.getUncertainty(0) == .05, "uncertainty preserved"); + check(fitter.fit.getParameterValue(0) == 2, "parameter value preserved"); + allowEditor[0] = true; + MouseEvent left = new MouseEvent(table, MouseEvent.MOUSE_PRESSED, 0, 0, 5, 5, 1, false, MouseEvent.BUTTON1); + check(table.editCellAt(0, 2, left), "left-click editing still available"); + table.getCellEditor().cancelCellEditing(); + if (args.length > 0) { + JFrame frame = new JFrame(); + frame.setContentPane(fitter); + frame.setSize(1000, 250); + frame.addNotify(); + frame.validate(); + java.awt.image.BufferedImage image = new java.awt.image.BufferedImage(1000, 250, java.awt.image.BufferedImage.TYPE_INT_RGB); + java.awt.Graphics2D graphics = image.createGraphics(); + frame.getContentPane().printAll(graphics); + graphics.dispose(); + try { javax.imageio.ImageIO.write(image, "png", new java.io.File(args[0])); } + catch (java.io.IOException ex) { throw new RuntimeException(ex); } + } + System.out.println("Passed: " + passed); + } finally { + for (Window window : Window.getWindows()) window.dispose(); + } + }); + System.exit(0); + } + static String labels(java.awt.Container parent) { + StringBuilder text = new StringBuilder(); + for (java.awt.Component child : parent.getComponents()) { + if (child instanceof JLabel) text.append(((JLabel)child).getText()).append("\n"); + if (child instanceof java.awt.Container) text.append(labels((java.awt.Container)child)); + } + return text.toString(); + } + static void reject(JTable table, int column, int button, int modifiers, boolean popup, String label) { + MouseEvent event = new MouseEvent(table, MouseEvent.MOUSE_PRESSED, 0, modifiers, 5, 5, 1, popup, button); + check(!table.editCellAt(0, column, event) && !table.isEditing(), label + " column " + column); + } + static void check(boolean ok, String message) { + if (!ok) throw new AssertionError(message); + passed++; + System.out.println("PASS: " + message); + } +} diff --git a/test/org/opensourcephysics/tools/CurveFitPrecisionTest.java b/test/org/opensourcephysics/tools/CurveFitPrecisionTest.java new file mode 100644 index 00000000..b641c520 --- /dev/null +++ b/test/org/opensourcephysics/tools/CurveFitPrecisionTest.java @@ -0,0 +1,59 @@ +package org.opensourcephysics.tools; + +import java.util.Locale; +import org.opensourcephysics.display.OSPRuntime; + +/** Standalone regression checks for reporting precision and retained values. */ +public class CurveFitPrecisionTest { + private static int passed; + + public static void main(String[] args) { + Locale original = Locale.getDefault(); + char separator = OSPRuntime.getCurrentDecimalSeparator(); + try { + OSPRuntime.setPreferredDecimalSeparator("."); + display(1.23456789, 0.056789, "1.235 ± 0.057"); + display(1.23456789, 0.05001, "1.235 ± 0.050"); + display(-1.23456789, 0.056789, "-1.235 ± 0.057"); + display(0, 0.056789, "0.000 ± 0.057"); + display(1.23456789, 0.0996, "1.23 ± 0.10"); + display(1.23456789, 0.996, "1.2 ± 1.0"); + display(1.23456789, 9.96, "(0.1 ± 1.0) E1"); + display(12345.6789, 567.89, "(1.235 ± 0.057) E4"); + display(0.00123456789, 0.000056789, "(1.235 ± 0.057) E-3"); + display(0.12, 5.6789, "0.1 ± 5.7"); + double value = 1.2345678901234567; + double sigma = 0.0567890123456789; + String[] text = DatasetCurveFitter.formatParameterWithUncertainty(value, sigma, 1); + String[] tooltip = text[1].split(" ± "); + check(Double.doubleToLongBits(Double.parseDouble(tooltip[0])) == Double.doubleToLongBits(value), "tooltip preserves parameter double"); + check(Double.doubleToLongBits(Double.parseDouble(tooltip[1])) == Double.doubleToLongBits(sigma), "tooltip preserves uncertainty double"); + DatasetCurveFitter.NumberField field = new DatasetCurveFitter.NumberField(12); + field.applyPattern("0.###"); + field.setValue(value); + check(Double.doubleToLongBits(field.getValue()) == Double.doubleToLongBits(value), "unchanged rounded editor retains original double"); + for (double unavailable : new double[] {0, -1, Double.NaN, Double.POSITIVE_INFINITY}) { + check(DatasetCurveFitter.formatParameterWithUncertainty(value, unavailable, 1) == null, "unavailable uncertainty " + unavailable); + } + Locale.setDefault(Locale.GERMANY); + display(1.23456789, 0.056789, "1.235 ± 0.057"); + OSPRuntime.setPreferredDecimalSeparator(","); + display(1.23456789, 0.056789, "1,235 ± 0,057"); + System.out.println("Passed: " + passed); + } finally { + Locale.setDefault(original); + OSPRuntime.setPreferredDecimalSeparator(String.valueOf(separator)); + } + } + + private static void display(double value, double sigma, String expected) { + String actual = DatasetCurveFitter.formatParameterWithUncertainty(value, sigma, 1)[0]; + check(expected.equals(actual), "expected " + expected + "; got " + actual); + } + + private static void check(boolean ok, String message) { + if (!ok) throw new AssertionError(message); + passed++; + System.out.println("PASS: " + message); + } +} diff --git a/test/org/opensourcephysics/tools/CurveFitReportTest.java b/test/org/opensourcephysics/tools/CurveFitReportTest.java new file mode 100644 index 00000000..e76ca002 --- /dev/null +++ b/test/org/opensourcephysics/tools/CurveFitReportTest.java @@ -0,0 +1,101 @@ +package org.opensourcephysics.tools; + +import org.opensourcephysics.display.Dataset; +import org.opensourcephysics.display.OSPRuntime; + +/** Analytic regression checks and tab-separated spreadsheet layout checks. */ +public class CurveFitReportTest { + static int passed; + public static void main(String[] args) throws Exception { + OSPRuntime.setPreferredDecimalSeparator("."); + KnownPolynomial line = new KnownPolynomial(new double[] {1, 2}); + Dataset data = new Dataset(); + data.setXYColumnNames("time", "position"); + // Residuals are orthogonal to the intercept and x: this is the exact OLS line. + data.append(new double[] {0, 1, 2, 3}, new double[] {1.1, 2.9, 4.9, 7.1}); + double[] stats = CurveFitReport.statistics(line, data, null, true); + close(stats[0], 4, "point count"); + close(stats[1], 2, "free parameters"); + close(stats[2], 2, "residual degrees of freedom"); + close(stats[3], .04, "SSE"); + close(stats[4], .1, "RMS uses n"); + close(stats[5], 1 - .04/20.04, "R-squared uses model residuals"); + close(stats[6], Math.sqrt(.02), "standard error uses n-p"); + close(stats[7], 20.04, "total sum of squares"); + String report = CurveFitReport.create(line, data, null, new double[] {.0632455532033676, .1183215956619923}, true, true); + String[] rows = report.split("\n"); + check(rows[0].startsWith("SUMMARY OUTPUT\tposition vs time\t"), "simple title and data-name fallback"); + check(rows[1].contains("position = ") && rows[1].contains("Automatic fit"), "second header contains model, equation, and mode"); + for (String row : rows) check(row.split("\t", -1).length == 5, "five aligned spreadsheet columns"); + check(report.indexOf("Regression Statistics") < report.indexOf("ANOVA") + && report.indexOf("ANOVA") < report.indexOf("Parameter\tCoefficients"), "Excel-style section order"); + close(cell(report, "Multiple R", 1), Math.sqrt(stats[5]), "Multiple R"); + close(cell(report, "Adjusted R Square", 1), 1 - (.04/2)/(20.04/3), "Adjusted R Square"); + close(cell(report, "Regression", 1), 1, "regression df"); + close(cell(report, "Regression", 2), 20, "regression SS"); + close(cell(report, "Regression", 3), 20, "regression MS"); + close(cell(report, "Regression", 4), 1000, "F statistic"); + close(cell(report, "Residual", 1), 2, "residual df"); + close(cell(report, "Residual", 2), .04, "residual SS"); + close(cell(report, "Residual", 3), .02, "residual MS"); + close(cell(report, "Total", 1), 3, "total df"); + check(report.split("Parameter\\tCoefficients", -1).length == 2, "coefficient header occurs once"); + check(!report.contains("rounded") && !report.contains("±"), "no repeated rounded coefficients"); + check(!report.toLowerCase().contains("weight"), "no irrelevant weights note"); + check(rows[rows.length - 1].startsWith("B\t"), "report ends with coefficients, without a prose footer"); + report = CurveFitReport.create(line, data, new boolean[] {true, false}, new double[] {.05, .01}, true, true); + close(cell(report, "Residual", 1), 3, "fixed parameter excluded from p"); + check(report.contains("A\t2.0\tN/A\tYes"), "fixed coefficient has no uncertainty"); + check(report.contains("Regression\tN/A\tN/A\tN/A\tN/A"), "no classical regression ANOVA for constrained fits"); + report = CurveFitReport.create(line, data, null, new double[] {.05, .01}, false, true); + check(report.contains("Manual parameters") && report.contains("A\t2.0\tN/A\tNo"), "manual mode does not reuse uncertainties"); + check(report.contains("Standard Error\tN/A"), "manual mode has no fitted residual standard error"); + KnownPolynomial parabola = new KnownPolynomial(new double[] {0, 0, 1}); + Dataset curved = new Dataset(); + curved.append(new double[] {-2, -1, 0, 1, 2}, new double[] {4, 1, 0, 1, 4}); + close(CurveFitReport.statistics(parabola, curved, null, true)[5], 1, "nonlinear curve has R-squared 1 despite zero linear correlation"); + UserFunction nonlinear = new UserFunction("Exponential"); + nonlinear.setParameters(new String[] {"A"}, new double[] {1}, null); + nonlinear.setExpression("exp(A*x)", new String[] {"x"}); + report = CurveFitReport.create(nonlinear, data, null, null, true, true); + check(report.contains("Regression\tN/A\tN/A\tN/A\tN/A"), "no classical regression ANOVA for nonlinear parameter models"); + Dataset constant = new Dataset(); + constant.append(new double[] {0, 1, 2}, new double[] {3, 3, 3}); + check(Double.isNaN(CurveFitReport.statistics(line, constant, null, true)[5]), "constant response has undefined R-squared"); + Dataset shortData = new Dataset(); shortData.append(0, 1); + check(Double.isNaN(CurveFitReport.statistics(line, shortData, null, true)[6]), "insufficient degrees of freedom"); + check(Double.isNaN(CurveFitReport.statistics(line, null, null, true)[3]), "empty dataset statistics unavailable"); + KnownPolynomial poor = new KnownPolynomial(new double[] {100,100}); + check(CurveFitReport.statistics(poor, data, null, false)[5] < 0, "negative R-squared retained"); + UserFunction invalid = new UserFunction("Invalid"); invalid.setExpression("sqrt(-1)", new String[] {"x"}); + check(Double.isNaN(CurveFitReport.statistics(invalid, data, null, false)[3]), "undefined model statistics unavailable"); + double value = 1.2345678901234567, sigma = .0567890123456789; + line.setParameterValue(0, value); + String params = CurveFitReport.create(line, data, null, new double[] {sigma,.01}, true, false); + String[] first = params.split("\n")[1].split("\t", -1); + check(Double.doubleToLongBits(Double.parseDouble(first[1])) == Double.doubleToLongBits(value), "coefficient round-trips exactly"); + check(Double.doubleToLongBits(Double.parseDouble(first[2])) == Double.doubleToLongBits(sigma), "uncertainty round-trips exactly"); + check(params.split("\n").length == 3, "parameters-only export has one header and two rows"); + check(first[3].equals("No") && first[4].isEmpty(), "coefficient and uncertainty each exported once"); + check(line.getParameterValue(0) == value, "copying leaves model unchanged"); + OSPRuntime.setPreferredDecimalSeparator(","); + params = CurveFitReport.create(line, data, null, new double[] {sigma,.01}, true, false); + check(params.contains("1,2345678901234567\t0,0567890123456789"), "locale decimal separator with tab delimiter"); + OSPRuntime.setPreferredDecimalSeparator("."); + if (args.length > 0) java.nio.file.Files.write(java.nio.file.Paths.get(args[0]), + CurveFitReport.create(new KnownPolynomial(new double[] {1,2}), data, null, + new double[] {.0632455532033676,.1183215956619923}, true, true).getBytes(java.nio.charset.StandardCharsets.UTF_8)); + System.out.println("Passed: " + passed); + } + static double cell(String report, String name, int column) { + for (String row : report.split("\n")) if (row.startsWith(name + "\t")) return Double.parseDouble(row.split("\t", -1)[column]); + throw new AssertionError("Missing row: " + name); + } + static void close(double actual, double expected, String message) { + check(Math.abs(actual - expected) < 1e-10 * Math.max(1,Math.abs(expected)), message + ": " + actual); + } + static void check(boolean ok, String message) { + if (!ok) throw new AssertionError(message); + passed++; System.out.println("PASS: " + message); + } +} From f0630515deb37ed67715f298645689ec62e1ccb2 Mon Sep 17 00:00:00 2001 From: paulnord Date: Sat, 5 Sep 2026 18:12:19 -0400 Subject: [PATCH 02/21] Test Control-click according to native platform behavior --- .../opensourcephysics/tools/CurveFitPopupTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/org/opensourcephysics/tools/CurveFitPopupTest.java b/test/org/opensourcephysics/tools/CurveFitPopupTest.java index 5d8a23b4..25cfa3d6 100644 --- a/test/org/opensourcephysics/tools/CurveFitPopupTest.java +++ b/test/org/opensourcephysics/tools/CurveFitPopupTest.java @@ -38,7 +38,16 @@ public static void main(String[] args) throws Exception { reject(table, column, MouseEvent.BUTTON3, 0, true, "right-click popup"); reject(table, column, MouseEvent.BUTTON3, 0, false, "right-click before popup release"); reject(table, column, MouseEvent.BUTTON1, InputEvent.CTRL_DOWN_MASK, true, "Control-click popup"); - reject(table, column, MouseEvent.BUTTON1, InputEvent.CTRL_DOWN_MASK, false, "macOS Control-click"); + if (org.opensourcephysics.display.OSPRuntime.isMac()) { + reject(table, column, MouseEvent.BUTTON1, InputEvent.CTRL_DOWN_MASK, false, "macOS Control-click"); + } else { + allowEditor[0] = true; + MouseEvent controlClick = new MouseEvent(table, MouseEvent.MOUSE_PRESSED, 0, + InputEvent.CTRL_DOWN_MASK, 5, 5, 1, false, MouseEvent.BUTTON1); + check(table.editCellAt(0, column, controlClick), "non-popup Control-click permits editing"); + table.getCellEditor().cancelCellEditing(); + allowEditor[0] = false; + } } check(fitter.isAutoFit(), "Autofit preserved"); check(fitter.getUncertainty(0) == .05, "uncertainty preserved"); From ed81ad23937685db541ec4f972ab58178c8f97dd Mon Sep 17 00:00:00 2001 From: paulnord Date: Sat, 5 Sep 2026 18:22:42 -0400 Subject: [PATCH 03/21] Support curve-fit reports and parameter sizing in SwingJS --- .../tools/CurveFitReport.java | 19 +++++++++----- .../tools/DatasetCurveFitter.java | 26 ++++++++++++++----- .../tools/CurveFitReportTest.java | 14 +++++++--- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/org/opensourcephysics/tools/CurveFitReport.java b/src/org/opensourcephysics/tools/CurveFitReport.java index c10dbc03..091c4b97 100644 --- a/src/org/opensourcephysics/tools/CurveFitReport.java +++ b/src/org/opensourcephysics/tools/CurveFitReport.java @@ -63,15 +63,20 @@ static String create(KnownFunction fit, Dataset data, boolean[] fixed, private static boolean hasDistinctAbscissas(Dataset data, int required) { java.util.HashSet distinct = new java.util.HashSet(); for (double x : data.getValidXPoints()) { - if (!Double.isFinite(x)) return false; + if (!isFinite(x)) return false; distinct.add(x == 0 ? 0.0 : x); if (distinct.size() >= required) return true; } return false; } + // SwingJS does not expose Java 8's Double.isFinite overload. + static boolean isFinite(double value) { + return !Double.isNaN(value) && !Double.isInfinite(value); + } + private static String integer(double value) { - return Double.isFinite(value) ? Integer.toString((int) value) : label("NA"); + return isFinite(value) ? Integer.toString((int) value) : label("NA"); } /** n, p, df, SSE, RMS, R-squared, residual standard error, SST; shared by screen and export. */ @@ -92,17 +97,17 @@ static double[] statistics(KnownFunction fit, Dataset data, boolean[] fixed, boo double delta = y[i] - mean; mean += delta / (i + 1); sst += delta * (y[i] - mean); - if (!Double.isFinite(x[i]) || !Double.isFinite(y[i]) || !Double.isFinite(predicted) + if (!isFinite(x[i]) || !isFinite(y[i]) || !isFinite(predicted) || (fit instanceof UserFunction && ((UserFunction) fit).evaluatedToNaN())) sse = Double.NaN; } int df = n - freeParameters; - boolean valid = n > 0 && Double.isFinite(sse); + boolean valid = n > 0 && isFinite(sse); return new double[] {n, freeParameters, autofit && n > 0 ? df : Double.NaN, valid ? sse : Double.NaN, valid ? Math.sqrt(sse / n) : Double.NaN, - valid && Double.isFinite(sst) && sst > 0 ? 1 - sse / sst : Double.NaN, + valid && isFinite(sst) && sst > 0 ? 1 - sse / sst : Double.NaN, valid && autofit && df > 0 ? Math.sqrt(sse / df) : Double.NaN, - n > 0 && Double.isFinite(sst) ? sst : Double.NaN}; + n > 0 && isFinite(sst) ? sst : Double.NaN}; } private static String label(String key) { @@ -110,7 +115,7 @@ private static String label(String key) { } private static String number(double value) { - return Double.isFinite(value) + return isFinite(value) ? Double.toString(value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : label("NA"); } diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index 13526e83..2329fbb9 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -583,12 +583,12 @@ private void refreshFitStatistics() { int index = VISIBLE_STATISTICS[i]; String key = CurveFitReport.STATISTIC_KEYS[index]; double value = values == null ? Double.NaN : values[index]; - String text = !Double.isFinite(value) ? ToolsRes.getString("DatasetCurveFitter.Report.NA") + String text = !CurveFitReport.isFinite(value) ? ToolsRes.getString("DatasetCurveFitter.Report.NA") : index < 3 ? Integer.toString((int) value) - : String.format(java.util.Locale.ROOT, "%.5g", value).replace('.', OSPRuntime.getCurrentDecimalSeparator()); + : formatRoot("%.5g", value).replace('.', OSPRuntime.getCurrentDecimalSeparator()); statisticLabels[i].setText(ToolsRes.getString("DatasetCurveFitter.Statistics." + key) + ": " + text); statisticLabels[i].setToolTipText(ToolsRes.getString("DatasetCurveFitter.Report." + key) + ": " - + (Double.isFinite(value) ? Double.toString(value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : text)); + + (CurveFitReport.isFinite(value) ? Double.toString(value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : text)); } } @@ -663,20 +663,30 @@ public String[] formatUncertainParameter(double value, double sigma, int extraPl return formatParameterWithUncertainty(value, sigma, extraPlaces); } + /** Locale-stable formatting supported by both Java and the SwingJS runtime. */ + private static String formatRoot(String pattern, double value) { + java.util.Formatter formatter = new java.util.Formatter(java.util.Locale.ROOT); + try { + return formatter.format(pattern, value).toString(); + } finally { + formatter.close(); + } + } + static String[] formatParameterWithUncertainty(double value, double sigma, int extraPlaces) { if (Double.isNaN(sigma) || Double.isInfinite(sigma) || sigma <= 0) { return null; } // Find the uncertainty exponent AFTER rounding, including carries such as // 0.0996 -> 0.10. Both displayed numbers must end at the same place. - String roundedSigma = String.format(java.util.Locale.ROOT, "%." + extraPlaces + "e", sigma); + String roundedSigma = formatRoot("%." + extraPlaces + "e", sigma); int expSig = Integer.parseInt(roundedSigma.substring(roundedSigma.indexOf('e') + 1)); int exp = value == 0 ? 0 : (int) Math.floor(Math.log10(Math.abs(value))); exp = Math.max(exp, expSig); int places = exp - expSig + extraPlaces; double scale = Math.pow(10, exp); - String val = String.format(java.util.Locale.ROOT, "%." + places + "f", value / scale); - String sig = String.format(java.util.Locale.ROOT, "%." + places + "f", sigma / scale); + String val = formatRoot("%." + places + "f", value / scale); + String sig = formatRoot("%." + places + "f", sigma / scale); String formatted = val + " \u00B1 " + sig; if (exp != 0) formatted = "(" + formatted + ") E" + exp; @@ -2027,7 +2037,9 @@ void sizeColumnsToContents() { .stringWidth(getColumnName(col)) + 16; if (col == 2) width = Math.max(width, getFontMetrics(getFont()) .stringWidth("(-0.00000 ± 0.00000) E-000") + 16); - for (int row = 0; row < getRowCount(); row++) { + // A detached SwingJS checkbox renderer cannot measure its DOM node. + // The fixed column already has room from its header and padding. + for (int row = 0; col != 1 && row < getRowCount(); row++) { Component renderer = prepareRenderer(getCellRenderer(row, col), row, col); width = Math.max(width, renderer.getPreferredSize().width + getIntercellSpacing().width + 12); } diff --git a/test/org/opensourcephysics/tools/CurveFitReportTest.java b/test/org/opensourcephysics/tools/CurveFitReportTest.java index e76ca002..1e7b4927 100644 --- a/test/org/opensourcephysics/tools/CurveFitReportTest.java +++ b/test/org/opensourcephysics/tools/CurveFitReportTest.java @@ -26,7 +26,7 @@ public static void main(String[] args) throws Exception { String[] rows = report.split("\n"); check(rows[0].startsWith("SUMMARY OUTPUT\tposition vs time\t"), "simple title and data-name fallback"); check(rows[1].contains("position = ") && rows[1].contains("Automatic fit"), "second header contains model, equation, and mode"); - for (String row : rows) check(row.split("\t", -1).length == 5, "five aligned spreadsheet columns"); + for (String row : rows) check(splitTabs(row).length == 5, "five aligned spreadsheet columns"); check(report.indexOf("Regression Statistics") < report.indexOf("ANOVA") && report.indexOf("ANOVA") < report.indexOf("Parameter\tCoefficients"), "Excel-style section order"); close(cell(report, "Multiple R", 1), Math.sqrt(stats[5]), "Multiple R"); @@ -72,7 +72,7 @@ public static void main(String[] args) throws Exception { double value = 1.2345678901234567, sigma = .0567890123456789; line.setParameterValue(0, value); String params = CurveFitReport.create(line, data, null, new double[] {sigma,.01}, true, false); - String[] first = params.split("\n")[1].split("\t", -1); + String[] first = splitTabs(params.split("\n")[1]); check(Double.doubleToLongBits(Double.parseDouble(first[1])) == Double.doubleToLongBits(value), "coefficient round-trips exactly"); check(Double.doubleToLongBits(Double.parseDouble(first[2])) == Double.doubleToLongBits(sigma), "uncertainty round-trips exactly"); check(params.split("\n").length == 3, "parameters-only export has one header and two rows"); @@ -88,12 +88,20 @@ public static void main(String[] args) throws Exception { System.out.println("Passed: " + passed); } static double cell(String report, String name, int column) { - for (String row : report.split("\n")) if (row.startsWith(name + "\t")) return Double.parseDouble(row.split("\t", -1)[column]); + for (String row : report.split("\n")) if (row.startsWith(name + "\t")) return Double.parseDouble(splitTabs(row)[column]); throw new AssertionError("Missing row: " + name); } static void close(double actual, double expected, String message) { check(Math.abs(actual - expected) < 1e-10 * Math.max(1,Math.abs(expected)), message + ": " + actual); } + // Preserve trailing empty TSV cells even in SwingJS, whose String.split + // implementation does not support Java's negative-limit behavior. + static String[] splitTabs(String row) { + String[] cells = (row + " ").split("\t"); + int last = cells.length - 1; + cells[last] = cells[last].substring(0, cells[last].length() - 1); + return cells; + } static void check(boolean ok, String message) { if (!ok) throw new AssertionError(message); passed++; System.out.println("PASS: " + message); From 9f4f430fb548f724c23d462d6e2ac1c63265df11 Mon Sep 17 00:00:00 2001 From: paulnord Date: Sat, 5 Sep 2026 18:41:36 -0400 Subject: [PATCH 04/21] Refresh constrained fits and distinguish unavailable uncertainties --- .../tools/DatasetCurveFitter.java | 40 ++++-- test/README.md | 6 +- .../tools/CurveFitConstraintTest.java | 120 ++++++++++++++++++ 3 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 test/org/opensourcephysics/tools/CurveFitConstraintTest.java diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index 2329fbb9..b71772e8 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -464,7 +464,8 @@ public double fit(KnownFunction fit, boolean fromScratch) { if (nothingToTest) { setUncertainties(null); - tab.refreshPlot(); + if (tab != null) + tab.refreshPlot(); drawer.functionChanged = true; paramTable.repaint(); } @@ -552,7 +553,7 @@ public double fit(KnownFunction fit, boolean fromScratch) { if (devSq == 0) { devSq = getDevSquared(fit, x, y); } - double rmsDev = fit.getParameterCount() > x.length && autofit? + double rmsDev = getFreeParameterCount(fit) > x.length && autofit? Double.NaN: Math.sqrt(devSq / x.length); @@ -1483,6 +1484,13 @@ private double[][] getUncertainties(KnownFunction original, KnownFunction fitted return null; double minChiSquared = calibrateChiSquared(fitted, x, y); + // A perfect fit has zero residual variance. Test identifiability using + // unscaled residuals so zero variance does not cause division by zero. + boolean perfectFit = sigma_y_squared == 0; + if (perfectFit) { + sigma_y_squared = 1; + minChiSquared = 0; + } int paramCount = original.getParameterCount(); double[] params = new double[paramCount]; @@ -1506,7 +1514,8 @@ private double[][] getUncertainties(KnownFunction original, KnownFunction fitted ArrayList results = new ArrayList(); // for each parameter in fitted function, measure curvature of chi squared to get sigma - double[] sigmas = new double[paramCount]; + double[] sigmas = new double[paramCount]; + java.util.Arrays.fill(sigmas, Double.NaN); for (int i = 0; i < paramCount; i++) { if (fittedParamIndex[i] < 0) { @@ -1542,9 +1551,10 @@ else if (twiceDeltaChiSq > 2) twiceDeltaChiSq = chiSq - 2 * minChiSquared; tries ++; } - if (twiceDeltaChiSq > 0) { // success: positive curvature so can determine sigma + if (CurveFitReport.isFinite(twiceDeltaChiSq) && twiceDeltaChiSq >= 0.001) { + // Only report an uncertainty when the profile has measurable curvature. // use eqn 8.13 in Data Reduction and Error Analysis For the Physical Sciences - sigmas[i] = delta * Math.sqrt(2 / twiceDeltaChiSq); + sigmas[i] = perfectFit ? 0 : delta * Math.sqrt(2 / twiceDeltaChiSq); // offset fixed parameter by +/- sigma and save test parameters for drawer for (int j = 0; j < 2; j++) { @@ -1831,6 +1841,16 @@ private UserFunction getTestFunction(KnownFunction f, String paramName, double p return testFunction; } + /** Counts the parameters the optimizer can change. */ + private int getFreeParameterCount(KnownFunction function) { + boolean[] fixed = fixedParams.get(function); + int count = 0; + for (int i = 0; i < function.getParameterCount(); i++) { + if (fixed == null || i >= fixed.length || !fixed[i]) count++; + } + return count; + } + /** * Gets a test function that mimics an input function but fixes some parameters. * @param f a KnownFunction @@ -2168,7 +2188,7 @@ else if (col == 1) { // if insufficient points to do fit return NaN if (dataset == null || - (autofit && fit.getParameterCount() > dataset.getValidXPoints().length)) + (autofit && getFreeParameterCount(fit) > dataset.getValidXPoints().length)) return Double.NaN; return Double.valueOf(fit.getParameterValue(row)); @@ -2178,8 +2198,12 @@ else if (col == 1) { public void setValueAt(Object value, int row, int col) { if (col == 1) { boolean[] fixed = fixedParams.get(fit); - if (fixed != null && fixed.length > row) { - fixed[row] = (Boolean)value; + if (fixed != null && fixed.length > row && fixed[row] != (Boolean) value) { + fixed[row] = (Boolean) value; + // Constraint changes invalidate the previous covariance estimates. + setUncertainties(null); + fit(fit); + fireTableRowsUpdated(0, getRowCount() - 1); } } } diff --git a/test/README.md b/test/README.md index 025e6f9d..a13d2dd7 100644 --- a/test/README.md +++ b/test/README.md @@ -8,7 +8,7 @@ with the built JAR: ```sh mkdir -p /tmp/osp-fit-tests javac -cp /path/to/osp.jar -d /tmp/osp-fit-tests test/org/opensourcephysics/tools/*.java -for test in CurveFitPrecisionTest CurveFitReportTest CurveFitPopupTest CurveFitDataToolLayoutTest; do +for test in CurveFitPrecisionTest CurveFitReportTest CurveFitPopupTest CurveFitDataToolLayoutTest CurveFitConstraintTest; do java -cp /tmp/osp-fit-tests:/path/to/osp.jar org.opensourcephysics.tools.$test || exit 1 done ``` @@ -19,3 +19,7 @@ visibility, compact statistics, and stable plot height across point selections. The precision and report tests cover display rounding, full-precision values, spreadsheet columns, and unavailable statistics. The popup test checks that context-menu gestures preserve Autofit while left-click editing remains available. + +The constraint test checks immediate refitting and report updates after fixed +checkbox edits, manual-mode preservation, one-point constrained fits, and unknown +uncertainties for non-identifiable models (including perfect fits). diff --git a/test/org/opensourcephysics/tools/CurveFitConstraintTest.java b/test/org/opensourcephysics/tools/CurveFitConstraintTest.java new file mode 100644 index 00000000..b2e3fe67 --- /dev/null +++ b/test/org/opensourcephysics/tools/CurveFitConstraintTest.java @@ -0,0 +1,120 @@ +package org.opensourcephysics.tools; + +import java.awt.Component; +import java.awt.Container; +import java.awt.Window; +import javax.swing.*; +import org.opensourcephysics.display.Dataset; + +/** Exercises constraint edits, identifiability, and constrained short datasets. */ +public class CurveFitConstraintTest { + static int passed; + + public static void main(String[] args) throws Exception { + SwingUtilities.invokeAndWait(() -> { + try { + runTests(); + System.out.println("Passed: " + passed); + } finally { + for (Window window : Window.getWindows()) window.dispose(); + } + }); + } + + static void runTests() { + Dataset noisy = data(new double[] {0,1,2,3}, new double[] {1.1,2.9,4.9,7.1}); + DatasetCurveFitter fitter = line(noisy); + close(fitter.getUncertainty(0), Math.sqrt(.02/5), "unconstrained slope uncertainty"); + close(fitter.getUncertainty(1), Math.sqrt(.014), "unconstrained intercept uncertainty"); + JTable table = table(fitter); + toggle(table, 0); + check(fitter.isAutoFit(), "fixing slope preserves Autofit"); + check(Double.isNaN(fitter.getUncertainty(0)), "fixed slope has no uncertainty"); + close(fitter.getUncertainty(1), Math.sqrt(.04/3/4), "free intercept uncertainty recomputed immediately"); + check(labels(fitter).contains("Free parameters: 1"), "free count refreshed immediately"); + check(labels(fitter).contains("Degrees of freedom: 3"), "degrees of freedom refreshed immediately"); + String report = report(fitter); + check(report.contains("A\t2.0\tN/A\tYes"), "export identifies fixed slope"); + close(CurveFitReportTest.cell(report, "B", 2), Math.sqrt(.04/3/4), "export uses new intercept uncertainty"); + + toggle(table, 1); + check(labels(fitter).contains("Free parameters: 0"), "all fixed updates free count"); + check(labels(fitter).contains("Degrees of freedom: 4"), "all fixed updates degrees of freedom"); + check(Double.isNaN(fitter.getUncertainty(0)) && Double.isNaN(fitter.getUncertainty(1)), "all fixed clears uncertainties"); + toggle(table, 1); + close(fitter.getUncertainty(1), Math.sqrt(.04/3/4), "unfixing intercept recomputes uncertainty"); + + fitter.setData(data(new double[] {1}, new double[] {2.9}), true); + close(((Number)table.getValueAt(0,2)).doubleValue(), 2, "fixed slope remains visible with one point"); + close(((Number)table.getValueAt(1,2)).doubleValue(), .9, "fitted intercept visible with one point"); + close(fitter.fit(fitter.fit), 0, "one free parameter and one point has finite RMS"); + check(Double.isNaN(fitter.getUncertainty(1)), "no residual degrees of freedom means unknown uncertainty"); + close(CurveFitReportTest.cell(report(fitter), "B", 1), .9, "screen and exported intercept agree"); + toggle(table, 0); + check(Double.isNaN(((Number)table.getValueAt(0,2)).doubleValue()), "two free parameters still require two points"); + check(Double.isNaN(fitter.fit(fitter.fit)), "underdetermined fit retains unavailable RMS"); + + fitter = line(noisy); + fitter.setAutoFit(false); + fitter.fit.setParameterValue(0, 5); + fitter.fit(fitter.fit); + toggle(table(fitter), 0); + check(!fitter.isAutoFit(), "constraint edit preserves manual mode"); + close(fitter.fit.getParameterValue(0), 5, "manual constraint does not alter slope"); + close(fitter.fit.getParameterValue(1), 1, "manual constraint does not optimize intercept"); + check(report(fitter).contains("Manual parameters"), "manual mode retained in report"); + + fitter = line(data(new double[] {0,1,2,3}, new double[] {1,3,5,7})); + close(fitter.getUncertainty(0), 0, "identifiable perfect-fit slope retains zero uncertainty"); + close(fitter.getUncertainty(1), 0, "identifiable perfect-fit intercept retains zero uncertainty"); + check(report(fitter).contains("A\t2.0\t0.0\tNo"), "legitimate zero uncertainty exported"); + for (double[] y : new double[][] {{.1,1.9,3.9,6.1}, {0,2,4,6}}) { + fitter = line(data(new double[] {0,1,2,3}, y)); + UserFunction redundant = new UserFunction("Redundant"); + redundant.setParameters(new String[] {"A","B"}, new double[] {1,1}, null); + redundant.setExpression("(A+B)*x", new String[] {"x"}); + // Exercise the same optimizer path with a deliberately non-identifiable model. + fitter.fit = redundant; + fitter.fit(redundant, true); + check(Double.isNaN(fitter.getUncertainty(0)), "unidentifiable A uncertainty unavailable"); + check(Double.isNaN(fitter.getUncertainty(1)), "unidentifiable B uncertainty unavailable"); + String[] rows = report(fitter).split("\n"); + for (String row : rows) { + if (row.startsWith("A\t") || row.startsWith("B\t")) + check(CurveFitReportTest.splitTabs(row)[2].equals("N/A"), "unidentifiable uncertainty exported as N/A"); + } + } + } + + static DatasetCurveFitter line(Dataset data) { + DatasetCurveFitter fitter = new DatasetCurveFitter(data, new FitBuilder(null)); + fitter.selectFit(fitter.getPolyFitNameOfDegree(1)); + fitter.setActiveAndFit(true); + return fitter; + } + static Dataset data(double[] x, double[] y) { + Dataset data = new Dataset(); data.setXYColumnNames("t", "y"); data.append(x,y); return data; + } + static JTable table(DatasetCurveFitter fitter) { + return (JTable)((JScrollPane)fitter.splitPane.getRightComponent()).getViewport().getView(); + } + static void toggle(JTable table, int row) { + check(table.editCellAt(row,1), "fixed checkbox editable"); + ((JCheckBox)table.getEditorComponent()).doClick(); + } + static String report(DatasetCurveFitter fitter) { + int count=fitter.fit.getParameterCount(); boolean[] fixed=new boolean[count]; double[] sigma=new double[count]; + for(int i=0;i Date: Sat, 5 Sep 2026 18:46:04 -0400 Subject: [PATCH 05/21] Run numerical constraint checks in the SwingJS test fixture --- .../tools/CurveFitConstraintTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/org/opensourcephysics/tools/CurveFitConstraintTest.java b/test/org/opensourcephysics/tools/CurveFitConstraintTest.java index b2e3fe67..c5e344ba 100644 --- a/test/org/opensourcephysics/tools/CurveFitConstraintTest.java +++ b/test/org/opensourcephysics/tools/CurveFitConstraintTest.java @@ -99,8 +99,15 @@ static JTable table(DatasetCurveFitter fitter) { return (JTable)((JScrollPane)fitter.splitPane.getRightComponent()).getViewport().getView(); } static void toggle(JTable table, int row) { - check(table.editCellAt(row,1), "fixed checkbox editable"); - ((JCheckBox)table.getEditorComponent()).doClick(); + if (org.opensourcephysics.display.OSPRuntime.isJS) { + // This numerical fixture has no DOM. Live browser checkbox editing + // is exercised separately in the attached Data Tool UI test. + check(table.isCellEditable(row,1), "fixed checkbox editable"); + table.getModel().setValueAt(!(Boolean)table.getValueAt(row,1),row,1); + } else { + check(table.editCellAt(row,1), "fixed checkbox editable"); + ((JCheckBox)table.getEditorComponent()).doClick(); + } } static String report(DatasetCurveFitter fitter) { int count=fitter.fit.getParameterCount(); boolean[] fixed=new boolean[count]; double[] sigma=new double[count]; From 629b46833f3ae59d3f8c412be1c263900d071ebc Mon Sep 17 00:00:00 2001 From: paulnord Date: Sat, 5 Sep 2026 19:05:57 -0400 Subject: [PATCH 06/21] Use effective parameter rank in fit statistics and clarify R squared --- .../resources/tools/tools.properties | 4 +- .../tools/CurveFitReport.java | 79 +++++++++++++++++-- test/README.md | 11 +++ .../tools/CurveFitReportTest.java | 32 +++++++- 4 files changed, 117 insertions(+), 9 deletions(-) diff --git a/src/org/opensourcephysics/resources/tools/tools.properties b/src/org/opensourcephysics/resources/tools/tools.properties index 94d28661..ad4ea0cb 100644 --- a/src/org/opensourcephysics/resources/tools/tools.properties +++ b/src/org/opensourcephysics/resources/tools/tools.properties @@ -946,8 +946,8 @@ DatasetCurveFitter.Statistics.SE=Residual SE DatasetCurveFitter.Report.Summary=SUMMARY OUTPUT DatasetCurveFitter.Report.RegressionStatistics=Regression Statistics DatasetCurveFitter.Report.MultipleR=Multiple R -DatasetCurveFitter.Report.RSquare=R Square -DatasetCurveFitter.Report.AdjustedRSquare=Adjusted R Square +DatasetCurveFitter.Report.RSquare=R Square (centered) +DatasetCurveFitter.Report.AdjustedRSquare=Adjusted R Square (centered) DatasetCurveFitter.Report.StandardError=Standard Error DatasetCurveFitter.Report.Observations=Observations DatasetCurveFitter.Report.ANOVA=ANOVA diff --git a/src/org/opensourcephysics/tools/CurveFitReport.java b/src/org/opensourcephysics/tools/CurveFitReport.java index 091c4b97..02f9e9dd 100644 --- a/src/org/opensourcephysics/tools/CurveFitReport.java +++ b/src/org/opensourcephysics/tools/CurveFitReport.java @@ -23,7 +23,9 @@ static String create(KnownFunction fit, Dataset data, boolean[] fixed, double[] stats = statistics(fit, data, fixed, autofit); double n = stats[0], p = stats[1], df = stats[2], sse = stats[3], r2 = stats[5], sst = stats[7]; row(out, label("RegressionStatistics")); - row(out, label("MultipleR"), number(r2 >= 0 ? Math.sqrt(r2) : Double.NaN)); + row(out, label("MultipleR"), number(autofit && fit instanceof KnownPolynomial + && p == fit.getParameterCount() && stats[8] == p && r2 >= 0 + ? Math.sqrt(r2) : Double.NaN)); row(out, label("RSquare"), number(r2)); row(out, label("AdjustedRSquare"), number(autofit && df > 0 && n > 1 && sst > 0 ? 1 - (sse / df) / (sst / (n - 1)) : Double.NaN)); @@ -35,7 +37,7 @@ static String create(KnownFunction fit, Dataset data, boolean[] fixed, // Classical regression ANOVA applies to an automatically fitted, // full-rank polynomial with a freely fitted intercept and no constraints. boolean regression = autofit && fit instanceof KnownPolynomial && p == fit.getParameterCount() - && df > 0 && p > 1 && sst > 0 && sse >= 0 && sse <= sst + && stats[8] == p && df > 0 && p > 1 && sst > 0 && sse >= 0 && sse <= sst && hasDistinctAbscissas(data, (int) p); double regressionDF = regression ? p - 1 : Double.NaN; double regressionSS = regression ? sst - sse : Double.NaN; @@ -79,7 +81,7 @@ private static String integer(double value) { return isFinite(value) ? Integer.toString((int) value) : label("NA"); } - /** n, p, df, SSE, RMS, R-squared, residual standard error, SST; shared by screen and export. */ + /** n, p, df, SSE, RMS, R-squared, residual standard error, SST, local rank; shared by screen and export. */ static double[] statistics(KnownFunction fit, Dataset data, boolean[] fixed, boolean autofit) { int freeParameters = 0; for (int i = 0; i < fit.getParameterCount(); i++) @@ -101,13 +103,78 @@ static double[] statistics(KnownFunction fit, Dataset data, boolean[] fixed, boo || (fit instanceof UserFunction && ((UserFunction) fit).evaluatedToNaN())) sse = Double.NaN; } - int df = n - freeParameters; + int rank = autofit && n > 0 ? parameterRank(fit, fixed, x) : -1; + int df = rank >= 0 ? n - rank : -1; boolean valid = n > 0 && isFinite(sse); - return new double[] {n, freeParameters, autofit && n > 0 ? df : Double.NaN, + return new double[] {n, freeParameters, autofit && n > 0 && rank >= 0 ? df : Double.NaN, valid ? sse : Double.NaN, valid ? Math.sqrt(sse / n) : Double.NaN, valid && isFinite(sst) && sst > 0 ? 1 - sse / sst : Double.NaN, valid && autofit && df > 0 ? Math.sqrt(sse / df) : Double.NaN, - n > 0 && isFinite(sst) ? sst : Double.NaN}; + n > 0 && isFinite(sst) ? sst : Double.NaN, rank}; + } + + /** Numerical rank of the free-parameter Jacobian, evaluated on a clone. + * Column normalization makes the tolerance independent of parameter units. + * For nonlinear models this is a local, linearized degrees-of-freedom estimate. + */ + private static int parameterRank(KnownFunction fit, boolean[] fixed, double[] x) { + KnownFunction probe = fit.clone(); + double[][] columns = new double[fit.getParameterCount()][x.length]; + int count = 0; + for (int j = 0; j < fit.getParameterCount(); j++) { + if (fixed != null && j < fixed.length && fixed[j]) continue; + double value = fit.getParameterValue(j); + double step = 1e-5 * Math.max(1, Math.abs(value)); + double norm = 0; + for (int i = 0; i < x.length; i++) { + if (!isFinite(x[i])) return -1; + double derivative; + if (fit instanceof KnownPolynomial) { + // Polynomial parameters run from highest power to constant. + // Exact derivatives avoid cancellation from large coefficients. + derivative = Math.pow(x[i], fit.getParameterCount() - 1 - j); + } else { + probe.setParameterValue(j, value + step); + double high = probe.evaluate(x[i]); + if (probe instanceof UserFunction && ((UserFunction) probe).evaluatedToNaN()) return -1; + probe.setParameterValue(j, value - step); + double low = probe.evaluate(x[i]); + if (probe instanceof UserFunction && ((UserFunction) probe).evaluatedToNaN()) return -1; + derivative = (high - low) / (2 * step); + } + if (!isFinite(derivative)) return -1; + columns[count][i] = derivative; + norm = Math.hypot(norm, derivative); + } + probe.setParameterValue(j, value); + if (norm > 0) { + for (int i = 0; i < x.length; i++) columns[count][i] /= norm; + count++; + } + } + // Pivoted, reorthogonalized Gram-Schmidt on normalized columns. + int rank = 0; + while (rank < count && rank < x.length) { + int pivot = rank; + double largest = 0; + for (int j = rank; j < count; j++) { + double norm = 0; + for (double v : columns[j]) norm = Math.hypot(norm, v); + if (norm > largest) { largest = norm; pivot = j; } + } + if (largest < 1e-7) break; + double[] swap = columns[rank]; columns[rank] = columns[pivot]; columns[pivot] = swap; + for (int i = 0; i < x.length; i++) columns[rank][i] /= largest; + for (int j = rank + 1; j < count; j++) { + for (int pass = 0; pass < 2; pass++) { + double dot = 0; + for (int i = 0; i < x.length; i++) dot += columns[rank][i] * columns[j][i]; + for (int i = 0; i < x.length; i++) columns[j][i] -= dot * columns[rank][i]; + } + } + rank++; + } + return rank; } private static String label(String key) { diff --git a/test/README.md b/test/README.md index a13d2dd7..729169d8 100644 --- a/test/README.md +++ b/test/README.md @@ -23,3 +23,14 @@ context-menu gestures preserve Autofit while left-click editing remains availabl The constraint test checks immediate refitting and report updates after fixed checkbox edits, manual-mode preservation, one-point constrained fits, and unknown uncertainties for non-identifiable models (including perfect fits). + +Statistical conventions: R Square and Adjusted R Square use centered total +sums of squares, including fits through zero (Excel uses an uncentered total +for those fits). Residual degrees of freedom use the numerical rank of the +free-parameter Jacobian; the displayed free count still counts editable parameters. +Rank uses normalized columns and a tolerance of 1e-7; for nonlinear models this +is a local linear approximation, not proof of global identifiability. +Multiple R and classical regression ANOVA are restricted to full-rank, +unconstrained automatic polynomial fits. Parameter standard errors retain +Tracker's profile-curvature method, which may differ from Jacobian covariance +estimates for nonlinear models; they are not 95% confidence intervals. diff --git a/test/org/opensourcephysics/tools/CurveFitReportTest.java b/test/org/opensourcephysics/tools/CurveFitReportTest.java index 1e7b4927..8cd130f5 100644 --- a/test/org/opensourcephysics/tools/CurveFitReportTest.java +++ b/test/org/opensourcephysics/tools/CurveFitReportTest.java @@ -30,7 +30,7 @@ public static void main(String[] args) throws Exception { check(report.indexOf("Regression Statistics") < report.indexOf("ANOVA") && report.indexOf("ANOVA") < report.indexOf("Parameter\tCoefficients"), "Excel-style section order"); close(cell(report, "Multiple R", 1), Math.sqrt(stats[5]), "Multiple R"); - close(cell(report, "Adjusted R Square", 1), 1 - (.04/2)/(20.04/3), "Adjusted R Square"); + close(cell(report, "Adjusted R Square (centered)", 1), 1 - (.04/2)/(20.04/3), "Adjusted R Square"); close(cell(report, "Regression", 1), 1, "regression df"); close(cell(report, "Regression", 2), 20, "regression SS"); close(cell(report, "Regression", 3), 20, "regression MS"); @@ -69,6 +69,36 @@ public static void main(String[] args) throws Exception { check(CurveFitReport.statistics(poor, data, null, false)[5] < 0, "negative R-squared retained"); UserFunction invalid = new UserFunction("Invalid"); invalid.setExpression("sqrt(-1)", new String[] {"x"}); check(Double.isNaN(CurveFitReport.statistics(invalid, data, null, false)[3]), "undefined model statistics unavailable"); + UserFunction redundant = new UserFunction("Redundant"); + redundant.setParameters(new String[] {"A", "B"}, new double[] {1, 1}, null); + redundant.setExpression("(A+B)*x", new String[] {"x"}); + Dataset redundantData = new Dataset(); + redundantData.append(new double[] {0,1,2,3}, new double[] {.1,1.9,3.9,6.1}); + stats = CurveFitReport.statistics(redundant, redundantData, null, true); + close(stats[1], 2, "nominal free parameter count retained"); + close(stats[8], 1, "redundant parameters have rank one"); + close(stats[2], 3, "rank determines residual degrees of freedom"); + close(stats[6], Math.sqrt(.04/3), "rank-correct residual standard error"); + close(redundant.getParameterValue(0), 1, "rank calculation preserves original parameters"); + report = CurveFitReport.create(redundant, redundantData, null, null, true, true); + close(cell(report, "Residual", 1), 3, "export uses rank-correct df"); + close(cell(report, "Adjusted R Square (centered)", 1), stats[5], "adjusted R square uses effective rank"); + check(report.contains("Multiple R\tN/A"), "Multiple R withheld outside classical polynomial fits"); + UserFunction origin = new UserFunction("Origin"); + origin.setParameters(new String[] {"A"}, new double[] {17.0/7}, null); + origin.setExpression("A*x", new String[] {"x"}); + report = CurveFitReport.create(origin, data, null, null, true, true); + close(cell(report, "R Square (centered)", 1), .9267179925862561, "through-origin R square explicitly centered"); + stats = CurveFitReport.statistics(line, data, new boolean[] {true,true}, true); + close(stats[2], 4, "all-fixed model uses n residual degrees of freedom"); + Dataset repeated = new Dataset(); repeated.append(new double[] {1,1,1,1}, new double[] {1,2,3,4}); + close(CurveFitReport.statistics(line, repeated, null, true)[2], 3, "repeated x lowers polynomial rank"); + UserFunction scaled = new UserFunction("Scaled"); + scaled.setParameters(new String[] {"A","B"}, new double[] {1,1}, null); + scaled.setExpression("A*x+1e-9*B", new String[] {"x"}); + close(CurveFitReport.statistics(scaled, data, null, true)[8], 2, "rank tolerates different parameter units"); + KnownPolynomial large = new KnownPolynomial(new double[] {1e20, 1}); + close(CurveFitReport.statistics(large, data, null, true)[8], 2, "large coefficient does not hide polynomial rank"); double value = 1.2345678901234567, sigma = .0567890123456789; line.setParameterValue(0, value); String params = CurveFitReport.create(line, data, null, new double[] {sigma,.01}, true, false); From 2ca0d6687dac7bb97b46ea104fdc1da6c18696da Mon Sep 17 00:00:00 2001 From: paulnord Date: Sun, 6 Sep 2026 19:10:59 -0400 Subject: [PATCH 07/21] Add unit-aware physics fit reports and optional common measurement uncertainty --- .../opensourcephysics/display/DataTable.java | 14 +- .../opensourcephysics/display/ExportText.java | 30 ++++ .../resources/tools/tools.properties | 36 ++++- .../tools/CurveFitReport.java | 90 +++++++---- .../opensourcephysics/tools/DataToolTab.java | 16 ++ .../tools/DataToolTable.java | 6 + .../tools/DatasetCurveFitter.java | 105 +++++++++++-- .../tools/FitMetadataProvider.java | 13 ++ .../tools/FitUncertainty.java | 59 ++++++++ test/README.md | 13 +- test/fit-report-physics.md | 88 +++++++++++ .../tools/CurveFitConstraintTest.java | 4 +- .../tools/CurveFitDataToolLayoutTest.java | 3 + .../tools/CurveFitPhysicsTest.java | 140 ++++++++++++++++++ .../tools/CurveFitReportTest.java | 16 +- 15 files changed, 573 insertions(+), 60 deletions(-) create mode 100644 src/org/opensourcephysics/display/ExportText.java create mode 100644 src/org/opensourcephysics/tools/FitMetadataProvider.java create mode 100644 src/org/opensourcephysics/tools/FitUncertainty.java create mode 100644 test/fit-report-physics.md create mode 100644 test/org/opensourcephysics/tools/CurveFitPhysicsTest.java diff --git a/src/org/opensourcephysics/display/DataTable.java b/src/org/opensourcephysics/display/DataTable.java index 9e30e60c..6d9292cc 100644 --- a/src/org/opensourcephysics/display/DataTable.java +++ b/src/org/opensourcephysics/display/DataTable.java @@ -357,6 +357,15 @@ public void setFormatPattern(String columnName, String pattern) { * @param units the units string (may be null) * @param tootip the tooltip (may be null) */ + /** Returns existing renderer metadata; never interprets the column name. */ + public String getUnits(String columnName) { + if (columnName == null) return null; + int suffix=columnName.indexOf("_{ "); + String key=suffix>0?columnName.substring(0,suffix):columnName; + UnitRenderer renderer=unitRenderersByColumnName.get(key); + return renderer==null?null:renderer.units; + } + public void setUnits(String columnName, String units, String tooltip) { if (units == null) { unitRenderersByColumnName.remove(columnName); @@ -2643,7 +2652,10 @@ public StringBuffer getData(boolean asFormatted) { if (isRowNumberVisible() && selectedColumns[j] == 0) continue; String name = getColumnName(selectedColumns[j]); - name = TeXParser.removeSubscripting(name); + String units = getUnits(name); + name = units == null || units.trim().length() == 0 + ? ExportText.ascii(TeXParser.removeSubscripting(name)) + : ExportText.header(name, units); // if (name.startsWith(FunctionEditor.THETA)) { // for (int i = 0; i < selectedRows.length; i++) { // Object val = getFormattedValueAt(selectedRows[i], selectedColumns[j]); diff --git a/src/org/opensourcephysics/display/ExportText.java b/src/org/opensourcephysics/display/ExportText.java new file mode 100644 index 00000000..1978c6ac --- /dev/null +++ b/src/org/opensourcephysics/display/ExportText.java @@ -0,0 +1,30 @@ +package org.opensourcephysics.display; + +/** Plain-text notation for spreadsheet headers and expressions, never numeric cells. */ +public final class ExportText { + private ExportText() {} + public static String ascii(String text) { + if (text == null) return ""; + String s = text.replace("\\cdot", "*").replace("\\times", "*") + .replace("\\mu", "u").replace("\\theta", "theta").replace("\\sigma", "sigma") + .replace("\\alpha", "alpha").replace("\\beta", "beta").replace("\\omega", "omega") + .replace("\\Delta", "Delta").replace("\\pi", "pi") + .replace("·", "*").replace("×", "*").replace("−", "-") + .replace("µ", "u").replace("μ", "u").replace("θ", "theta") + .replace("σ", "sigma").replace("α", "alpha").replace("β", "beta") + .replace("ω", "omega").replace("Δ", "Delta").replace("π", "pi"); + String supers = "⁰¹²³⁴⁵⁶⁷⁸⁹⁻⁺", normal = "0123456789-+"; + StringBuilder out = new StringBuilder(); boolean exponent = false; + for (int i=0;i=0) { if(!exponent) out.append('^'); out.append(normal.charAt(n)); exponent=true; } + else { out.append(s.charAt(i)); exponent=false; } + } + return out.toString().replaceAll("([_^])\\{([^{}]*)\\}", "$1$2"); + } + public static String header(String name, String units) { + String label=ascii(name), unit=ascii(units).trim(); + String suffix=" ("+unit+")"; + return unit.length()==0 || label.endsWith(suffix) ? label : label+suffix; + } +} diff --git a/src/org/opensourcephysics/resources/tools/tools.properties b/src/org/opensourcephysics/resources/tools/tools.properties index ad4ea0cb..cad384a1 100644 --- a/src/org/opensourcephysics/resources/tools/tools.properties +++ b/src/org/opensourcephysics/resources/tools/tools.properties @@ -926,8 +926,8 @@ DatasetCurveFitter.Report.Yes=Yes DatasetCurveFitter.Report.No=No DatasetCurveFitter.Report.NA=N/A DatasetCurveFitter.Report.Points=Data points (n) -DatasetCurveFitter.Report.Free=Free parameters (p) -DatasetCurveFitter.Report.DF=Residual degrees of freedom (n - p) +DatasetCurveFitter.Report.Free=Free parameters +DatasetCurveFitter.Report.DF=Degrees of freedom (n - rank) DatasetCurveFitter.Report.SSE=Residual sum of squares (SSE) DatasetCurveFitter.Report.RMS=RMS deviation (sqrt(SSE/n)) DatasetCurveFitter.Report.R2=R-squared (1 - SSE/SST) @@ -955,3 +955,35 @@ DatasetCurveFitter.Report.Regression=Regression DatasetCurveFitter.Report.Residual=Residual DatasetCurveFitter.Report.Total=Total DatasetCurveFitter.Report.Coefficients=Coefficients + +DatasetCurveFitter.Uncertainty.Label=Data uncertainty: +DatasetCurveFitter.Uncertainty.Estimated=Estimated from residuals +DatasetCurveFitter.Uncertainty.Constant=Constant sigma_y +DatasetCurveFitter.Uncertainty.Pixels=Position sigma (pixels) +DatasetCurveFitter.Uncertainty.Custom=Choose a value or type a custom positive uncertainty. +DatasetCurveFitter.Uncertainty.Invalid=Unavailable: enter a positive value and check calibration. +DatasetCurveFitter.Report.Fit=FIT +DatasetCurveFitter.Report.XVariable=X variable +DatasetCurveFitter.Report.YVariable=Y variable +DatasetCurveFitter.Report.Rank=Independent parameter rank +DatasetCurveFitter.Report.Identifiability=Fit parameters are not independently identifiable +DatasetCurveFitter.Report.Residuals=RESIDUALS +DatasetCurveFitter.Report.RMSResidual=RMS residual +DatasetCurveFitter.Report.Goodness=GOODNESS OF FIT +DatasetCurveFitter.Report.ChiSquare=Chi square +DatasetCurveFitter.Report.ReducedChiSquare=Reduced chi square +DatasetCurveFitter.Report.ChiProbability=Chi-square probability Q +DatasetCurveFitter.Report.ByConstruction=by construction +DatasetCurveFitter.Report.UncertaintyModel=UNCERTAINTY MODEL +DatasetCurveFitter.Report.Estimated=Estimated from residuals +DatasetCurveFitter.Report.Specified=Specified constant sigma_y +DatasetCurveFitter.Report.EstimatedSigma=Estimated sigma_y +DatasetCurveFitter.Report.SpecifiedSigma=Specified sigma_y +DatasetCurveFitter.Report.SigmaPixels=Specified position uncertainty +DatasetCurveFitter.Report.EstimatedCaution=Residual-estimated reduced chi square is 1 by construction, not an independent goodness-of-fit test. Q is unavailable. A perfect fit has zero estimated sigma and undefined chi square. +DatasetCurveFitter.Report.QCaution=Q assumes independent Gaussian measurement errors with the stated common sigma. For nonlinear fits the rank-based degrees of freedom and Q are local approximations. +DatasetCurveFitter.Report.R2Description=Centered R Square describes variance explained, not proof of goodness of fit. Excel uses an uncentered convention for some fits through zero. +DatasetCurveFitter.Report.Parameters=PARAMETERS +DatasetCurveFitter.Report.Units=Units + +DatasetCurveFitter.Report.ResidualStandardError=Residual standard error diff --git a/src/org/opensourcephysics/tools/CurveFitReport.java b/src/org/opensourcephysics/tools/CurveFitReport.java index 02f9e9dd..8eb21aa6 100644 --- a/src/org/opensourcephysics/tools/CurveFitReport.java +++ b/src/org/opensourcephysics/tools/CurveFitReport.java @@ -10,27 +10,45 @@ private CurveFitReport() {} static String create(KnownFunction fit, Dataset data, boolean[] fixed, double[] uncertainties, boolean autofit, boolean includeStatistics) { - StringBuilder out = new StringBuilder(); - if (includeStatistics) { - String xName = data == null ? "x" : data.getXColumnName(); - String yName = data == null ? "y" : data.getYColumnName(); - String dataName = data == null ? "" : data.getName(); - if (dataName == null || dataName.trim().isEmpty()) dataName = yName + " vs " + xName; - row(out, label("Summary"), dataName); - row(out, label("Model"), fit.getName(), label("Equation"), - yName + " = " + fit.getExpression(xName), label(autofit ? "Auto" : "Manual")); - row(out); - double[] stats = statistics(fit, data, fixed, autofit); - double n = stats[0], p = stats[1], df = stats[2], sse = stats[3], r2 = stats[5], sst = stats[7]; - row(out, label("RegressionStatistics")); - row(out, label("MultipleR"), number(autofit && fit instanceof KnownPolynomial - && p == fit.getParameterCount() && stats[8] == p && r2 >= 0 - ? Math.sqrt(r2) : Double.NaN)); - row(out, label("RSquare"), number(r2)); - row(out, label("AdjustedRSquare"), number(autofit && df > 0 && n > 1 && sst > 0 - ? 1 - (sse / df) / (sst / (n - 1)) : Double.NaN)); - row(out, label("StandardError"), number(stats[6])); - row(out, label("Observations"), Integer.toString((int) n)); + return create(fit,data,fixed,uncertainties,autofit,includeStatistics,null,null,FitUncertainty.estimated(),Double.NaN); + } + static String create(KnownFunction fit, Dataset data, boolean[] fixed, + double[] uncertainties, boolean autofit, boolean includeStatistics, + String xUnits, String yUnits, FitUncertainty uncertainty, double unitsPerPixel) { + StringBuilder out=new StringBuilder(); + if(includeStatistics) { + String xName=data==null?"x":data.getXColumnName(), yName=data==null?"y":data.getYColumnName(); + row(out,label("Fit"),data==null?"":data.getName()); + row(out,label("Model"),fit.getName(),label(autofit?"Auto":"Manual")); + row(out,label("XVariable"),org.opensourcephysics.display.ExportText.header(xName,xUnits)); + row(out,label("YVariable"),org.opensourcephysics.display.ExportText.header(yName,yUnits)); + row(out,label("Equation"),org.opensourcephysics.display.ExportText.ascii(yName+" = "+fit.getExpression(xName))); + double[] stats=statistics(fit,data,fixed,autofit); + double n=stats[0],p=stats[1],df=stats[2],sse=stats[3],r2=stats[5],sst=stats[7]; + row(out,label("Observations"),integer(n)); + row(out,label("Free"),integer(p)); + row(out,label("Rank"),stats[8]<0?label("NA"):integer(stats[8])); + row(out,label("DF"),integer(df)); + if(autofit && stats[8]>=0 && stats[8]0 && n>1 && sst>0?1-(sse/df)/(sst/(n-1)):Double.NaN)); + row(out,label("MultipleR"),number(autofit && fit instanceof KnownPolynomial && p==fit.getParameterCount() + && stats[8]==p && r2>=0?Math.sqrt(r2):Double.NaN)); + double[] chi=uncertainty.statistics(stats,autofit,unitsPerPixel); + row(out,label("ChiSquare"),number(chi[1])); + row(out,label("ReducedChiSquare"),number(chi[2]),uncertainty.mode==FitUncertainty.ESTIMATED && isFinite(chi[2])?label("ByConstruction"):""); + row(out,label("ChiProbability"),number(chi[3])); + row(out,label("R2Description")); + row(out);row(out,label("UncertaintyModel"),label(uncertainty.mode==FitUncertainty.ESTIMATED?"Estimated":"Specified")); + row(out,label(uncertainty.mode==FitUncertainty.ESTIMATED?"EstimatedSigma":"SpecifiedSigma"),number(chi[0]),org.opensourcephysics.display.ExportText.ascii(yUnits)); + if(uncertainty.mode==FitUncertainty.PIXELS)row(out,label("SigmaPixels"),number(uncertainty.value),"pixels"); + if(uncertainty.mode==FitUncertainty.ESTIMATED)row(out,label("EstimatedCaution")); + else row(out,label("QCaution")); row(out); row(out, label("ANOVA")); row(out, "", "df", "SS", "MS", "F"); @@ -49,18 +67,28 @@ static String create(KnownFunction fit, Dataset data, boolean[] fixed, row(out, label("Total"), n > 0 ? Integer.toString((int) n - 1) : label("NA"), number(sst)); row(out); } - // Exactly one numeric coefficient and uncertainty per parameter. No - // duplicate rounded column: rounding belongs to the on-screen display. - row(out, label("Parameter"), label("Coefficients"), label("StandardError"), label("Fixed")); - for (int i = 0; i < fit.getParameterCount(); i++) { - boolean isFixed = fixed != null && i < fixed.length && fixed[i]; - double sigma = autofit && !isFixed && uncertainties != null && i < uncertainties.length - ? uncertainties[i] : Double.NaN; - if (sigma < 0) sigma = Double.NaN; - row(out, fit.getParameterName(i), number(fit.getParameterValue(i)), number(sigma), label(isFixed ? "Yes" : "No")); + if(includeStatistics)row(out,label("Parameters")); + row(out,label("Parameter"),label("Coefficients"),label("StandardError"),label("Units"),label("Fixed")); + for(int i=0;i distinct = new java.util.HashSet(); @@ -117,7 +145,7 @@ valid && isFinite(sst) && sst > 0 ? 1 - sse / sst : Double.NaN, * Column normalization makes the tolerance independent of parameter units. * For nonlinear models this is a local, linearized degrees-of-freedom estimate. */ - private static int parameterRank(KnownFunction fit, boolean[] fixed, double[] x) { + static int parameterRank(KnownFunction fit, boolean[] fixed, double[] x) { KnownFunction probe = fit.clone(); double[][] columns = new double[fit.getParameterCount()][x.length]; int count = 0; diff --git a/src/org/opensourcephysics/tools/DataToolTab.java b/src/org/opensourcephysics/tools/DataToolTab.java index 6db28890..9889b4e4 100644 --- a/src/org/opensourcephysics/tools/DataToolTab.java +++ b/src/org/opensourcephysics/tools/DataToolTab.java @@ -183,6 +183,15 @@ public DatasetCurveFitter getCurveFitter() { protected DatasetManager dataManager = new DatasetManager(); // datasets in this tab protected JSplitPane[] splitPanes; protected DataToolPlotter plot; + private FitMetadataProvider fitMetadataProvider; + /** Live metadata only; data values and saved project state are unaffected. */ + public void setFitMetadataProvider(FitMetadataProvider provider) { + fitMetadataProvider=provider; + if(curveFitter!=null)curveFitter.refreshUncertaintyModel(); + } + FitMetadataProvider getFitMetadataProvider() { return fitMetadataProvider; } + public void setColumnUnits(String name,String units) { dataTable.setUnits(name,units,null); } + protected DataToolTable dataTable; protected DataToolStatsTable statsTable; protected DataToolPropsTable propsTable; @@ -784,6 +793,13 @@ public void saveOwnedColumnNames(String columnOwnerName, Data data) { * @param ID the ID number of the desired column * @return the tab column name, or null if not found */ + /** Resolve a source column without confusing linked x and y columns sharing an ID. */ + public String getColumnName(int ID,int sourceColumn) { + for(Dataset column:dataManager.getDatasetsRaw()) + if(column.getID()==ID && column.getColumnID()==sourceColumn)return column.getYColumnName(); + return null; + } + public String getColumnName(int ID) { for (Dataset column : dataManager.getDatasetsRaw()) { if (column.getID() == ID) diff --git a/src/org/opensourcephysics/tools/DataToolTable.java b/src/org/opensourcephysics/tools/DataToolTable.java index ee1cff78..c907d84e 100644 --- a/src/org/opensourcephysics/tools/DataToolTable.java +++ b/src/org/opensourcephysics/tools/DataToolTable.java @@ -85,6 +85,12 @@ */ @SuppressWarnings("serial") public class DataToolTable extends DataTable { + @Override public String getUnits(String name) { + FitMetadataProvider provider=dataToolTab.getFitMetadataProvider(); + String units=provider==null?null:provider.getUnits(name); + return units==null?super.getUnits(name):units; + } + // static fields and constants protected final static int RENAME_COLUMN_EDIT = 0; protected final static int INSERT_COLUMN_EDIT = 1; diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index b71772e8..3249be95 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -197,6 +197,54 @@ public class DatasetCurveFitter extends JPanel { private DataToolTab tab; KnownFunction fit; // the function to fit to the data + private FitUncertainty uncertaintyModel=FitUncertainty.estimated(); + private JComboBox uncertaintyChoice, uncertaintyValue; + private JLabel uncertaintyStatus; + private boolean updatingUncertaintyControls; + public FitUncertainty getUncertaintyModel() { return uncertaintyModel; } + public void setUncertaintyModel(int mode,double value) { + uncertaintyModel=new FitUncertainty(mode,value); + refreshUncertaintyModel(); + } + private String variableUnits(boolean x) { + return tab==null || dataset==null ? null : tab.dataTable.getUnits(x?dataset.getXColumnName():dataset.getYColumnName()); + } + public double getYUnitsPerPixel() { + FitMetadataProvider provider=tab==null?null:tab.getFitMetadataProvider(); + return provider==null || dataset==null?Double.NaN:provider.getYUnitsPerPixel(dataset.getYColumnName()); + } + /** Recompute profile errors only: do not rerun the coefficient optimizer. */ + public void refreshUncertaintyModel() { + if(fit!=null && dataset!=null && drawer!=null) { + if(autofit) { + KnownFunction probe=fit.clone(); + KnownFunction free=getTestFunction(probe,fixedParams.get(fit)); + setUncertainties(getUncertainties(probe,free,dataset.getValidXPoints(),dataset.getValidYPoints())); + } else setUncertainties(null); + drawer.functionChanged=true; + paramTable.repaint(); + } + refreshFitStatistics(); + refreshUncertaintyControls(); + } + private void refreshUncertaintyControls() { + if(uncertaintyChoice==null)return; + updatingUncertaintyControls=true; + boolean pixels=FitUncertainty.positive(getYUnitsPerPixel()); + if((pixels || uncertaintyModel.mode==FitUncertainty.PIXELS) && uncertaintyChoice.getItemCount()==2) + uncertaintyChoice.addItem(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Pixels")); + if(!pixels && uncertaintyModel.mode!=FitUncertainty.PIXELS && uncertaintyChoice.getItemCount()==3) + uncertaintyChoice.removeItemAt(2); + uncertaintyChoice.setSelectedIndex(uncertaintyModel.mode); + uncertaintyValue.setEnabled(uncertaintyModel.mode!=FitUncertainty.ESTIMATED); + if(uncertaintyModel.mode!=FitUncertainty.ESTIMATED)uncertaintyValue.setSelectedItem(Double.toString(uncertaintyModel.value)); + double sigma=uncertaintyModel.sigma(Double.NaN,getYUnitsPerPixel()); + uncertaintyStatus.setText(uncertaintyModel.mode==FitUncertainty.ESTIMATED?"": + FitUncertainty.positive(sigma)?org.opensourcephysics.display.ExportText.ascii(variableUnits(false)): + ToolsRes.getString("DatasetCurveFitter.Uncertainty.Invalid")); + updatingUncertaintyControls=false; + } + double sigma_y_squared = 1; // an estimate of the SD in the y deviations from the fit ArrayList testFunctions = new ArrayList();; Color color = Color.MAGENTA; @@ -265,7 +313,7 @@ public void setAutoFit(boolean autofit) { private JComboBox fitDropDown; private JTextField eqnField; private NumberField rmsField; - private JPanel statisticsPanel; + private JPanel statisticsPanel, uncertaintyPanel; private JLabel[] statisticLabels; private static final int[] VISIBLE_STATISTICS = {0, 1, 2, 5, 3, 6}; private ParamTable paramTable; @@ -289,6 +337,7 @@ public JSplitPane getSplitPane() { */ public DatasetCurveFitter(Dataset data, FitBuilder builder) { dataset = data; + refreshUncertaintyControls(); fitBuilder = builder; createGUI(); fitBuilder.removePropertyChangeListener(fitListener); @@ -590,20 +639,24 @@ private void refreshFitStatistics() { statisticLabels[i].setText(ToolsRes.getString("DatasetCurveFitter.Statistics." + key) + ": " + text); statisticLabels[i].setToolTipText(ToolsRes.getString("DatasetCurveFitter.Report." + key) + ": " + (CurveFitReport.isFinite(value) ? Double.toString(value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : text)); - } - } + if(index==5)statisticLabels[i].setToolTipText(ToolsRes.getString("DatasetCurveFitter.Report.R2Description")); + } + } - /** Copies a snapshot of the current fit without fitting or rounding its data. */ + /** Copies a snapshot of the current fit without fitting or rounding its data. */ private void copyFitResults(boolean includeStatistics) { if (paramTable.isEditing() && !paramTable.getCellEditor().stopCellEditing()) return; if (fit == null) return; + // Calibration is live host metadata; refresh pixel-scaled profile errors + // before taking a snapshot, without changing the best-fit coefficients. + if (uncertaintyModel.mode == FitUncertainty.PIXELS) refreshUncertaintyModel(); double[] sigma = new double[fit.getParameterCount()]; for (int i = 0; i < sigma.length; i++) sigma[i] = getUncertainty(i); OSPRuntime.copy(CurveFitReport.create(fit, dataset, fixedParams.get(fit), sigma, - autofit, includeStatistics), null); + autofit, includeStatistics, variableUnits(true), variableUnits(false), uncertaintyModel, getYUnitsPerPixel()), null); } /** @@ -726,7 +779,7 @@ public Map getSelectedFitParameters() { public Dimension getMinimumSize() { if (statisticsPanel == null) return super.getMinimumSize(); return new Dimension(fitBar.getPreferredSize().width, - splitPane.getPreferredSize().height + statisticsPanel.getPreferredSize().height + 4); + splitPane.getPreferredSize().height + statisticsPanel.getPreferredSize().height + (uncertaintyPanel==null?0:uncertaintyPanel.getPreferredSize().height) + 4); } // _______________________ protected & private methods @@ -1090,6 +1143,24 @@ public void layoutContainer(Container target) { } JPanel statisticsContainer = new JPanel(new BorderLayout()); statisticsContainer.add(statisticsPanel, BorderLayout.NORTH); + uncertaintyPanel=new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT,4,2)); + uncertaintyPanel.add(new JLabel(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Label"))); + uncertaintyChoice=new JComboBox(new String[]{ToolsRes.getString("DatasetCurveFitter.Uncertainty.Estimated"),ToolsRes.getString("DatasetCurveFitter.Uncertainty.Constant")}); + uncertaintyValue=new JComboBox(new String[]{"0.5","1.0","1.5","2.0","2.5","3.0"}); + uncertaintyValue.setEditable(true);uncertaintyValue.setSelectedItem("1.0"); + uncertaintyValue.setToolTipText(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Custom")); + uncertaintyStatus=new JLabel(); + java.awt.event.ActionListener changed=e->{ + if(updatingUncertaintyControls)return; + double value=Double.NaN; + try { value=Double.parseDouble(uncertaintyValue.getEditor().getItem().toString().trim().replace(OSPRuntime.getCurrentDecimalSeparator(),'.')); } + catch(Exception ex) { /* Keep invalid supplied input unavailable, never estimate it. */ } + setUncertaintyModel(uncertaintyChoice.getSelectedIndex(),value); + }; + uncertaintyChoice.addActionListener(changed);uncertaintyValue.addActionListener(changed); + uncertaintyPanel.add(uncertaintyChoice);uncertaintyPanel.add(uncertaintyValue);uncertaintyPanel.add(uncertaintyStatus); + statisticsContainer.add(uncertaintyPanel,BorderLayout.SOUTH); + refreshUncertaintyControls(); add(statisticsContainer, BorderLayout.CENTER); refreshGUI(); // refreshFitDropDown(); @@ -1358,7 +1429,7 @@ int prepareFitLayout(int availableWidth) { int divider = stacked ? controlsHeight : Math.max(controlsWidth, Math.min(splitPane.getDividerLocation(), width - parameterWidth - splitPane.getDividerSize())); if (splitPane.getDividerLocation() != divider) splitPane.setDividerLocation(divider); - return infoHeight + statisticsPanel.getPreferredSize().height + 4; + return infoHeight + statisticsPanel.getPreferredSize().height + (uncertaintyPanel==null?0:uncertaintyPanel.getPreferredSize().height) + 4; } private void layoutFitPane() { @@ -1447,13 +1518,16 @@ private double getDevSquared(Function f, double[] x, double[] y) { * @param x the x data * @param y the y data * - * @return calibrated chi squared = x.length - paramCount + * @return chi squared with the selected residual-estimated or supplied scale */ private double calibrateChiSquared(KnownFunction f, double[] x, double[] y) { - int paramCount = f.getParameterCount(); - // set sigma so that chi squared is x.length - paramCount - sigma_y_squared = getDevSquared(f, x, y) / (x.length - paramCount); - return x.length - paramCount; + // Residual estimation uses the independent rank; supplied sigma is not rescaled. + int rank=CurveFitReport.parameterRank(f,null,x); + int df=rank<0?-1:x.length-rank; + double residual=getDevSquared(f,x,y); + double sigma=uncertaintyModel.sigma(df>0?Math.sqrt(residual/df):Double.NaN,getYUnitsPerPixel()); + sigma_y_squared=sigma*sigma; + return uncertaintyModel.mode==FitUncertainty.ESTIMATED?df:residual/sigma_y_squared; } /** @@ -1478,15 +1552,16 @@ private double getChiSquared(Function f, double[] x, double[] y) { * @return double[][] {{param uncertainties}, {fit params 1}, {fit params 2}, ...} */ private double[][] getUncertainties(KnownFunction original, KnownFunction fitted, double[] x, double[] y) { - // calibrate (set sigma_y_squared) and get min chi squared (= x.length-f.paramCount) + // Set the selected uncertainty scale and obtain the minimum chi squared. int fitCount = fitted.getParameterCount(); - if (fitCount == 0 || x.length - fitCount <= 0) + if (fitCount == 0 || x.length - CurveFitReport.parameterRank(fitted,null,x) <= 0) return null; double minChiSquared = calibrateChiSquared(fitted, x, y); // A perfect fit has zero residual variance. Test identifiability using // unscaled residuals so zero variance does not cause division by zero. - boolean perfectFit = sigma_y_squared == 0; + boolean perfectFit = uncertaintyModel.mode==FitUncertainty.ESTIMATED && sigma_y_squared == 0; + if (!perfectFit && !FitUncertainty.positive(sigma_y_squared)) return null; if (perfectFit) { sigma_y_squared = 1; minChiSquared = 0; diff --git a/src/org/opensourcephysics/tools/FitMetadataProvider.java b/src/org/opensourcephysics/tools/FitMetadataProvider.java new file mode 100644 index 00000000..a5359f23 --- /dev/null +++ b/src/org/opensourcephysics/tools/FitMetadataProvider.java @@ -0,0 +1,13 @@ +package org.opensourcephysics.tools; + +/** Optional live host metadata. No parsing of names and no serialized state. + * A future per-observation uncertainty provider can be added separately. + */ +public interface FitMetadataProvider { + /** Null means no host metadata: retain DataTable's existing units. */ + String getUnits(String columnName); + /** Physical y units per image pixel, only for a directly calibrated position. + * Return NaN for derived data, missing calibration, or varying conversion. + */ + double getYUnitsPerPixel(String columnName); +} diff --git a/src/org/opensourcephysics/tools/FitUncertainty.java b/src/org/opensourcephysics/tools/FitUncertainty.java new file mode 100644 index 00000000..e7ad507d --- /dev/null +++ b/src/org/opensourcephysics/tools/FitUncertainty.java @@ -0,0 +1,59 @@ +package org.opensourcephysics.tools; + +/** Session-only common data uncertainty. Invalid supplied input never becomes estimated input. */ +public final class FitUncertainty { + public static final int ESTIMATED=0, CONSTANT=1, PIXELS=2; + public final int mode; + public final double value; + public FitUncertainty(int mode, double value) { + if(mode<0 || mode>2)throw new IllegalArgumentException("Unknown uncertainty model"); + this.mode=mode;this.value=value; + } + public static FitUncertainty estimated() { return new FitUncertainty(ESTIMATED,Double.NaN); } + public double sigma(double residualSigma, double unitsPerPixel) { + if(mode==ESTIMATED)return residualSigma; + if(!positive(value))return Double.NaN; + return mode==CONSTANT ? value : positive(unitsPerPixel) && positive(value*unitsPerPixel) ? value*unitsPerPixel : Double.NaN; + } + static boolean positive(double x) { return CurveFitReport.isFinite(x) && x>0; } + /** sigma, chi-square, reduced chi-square, chi-square survival probability. */ + double[] statistics(double[] fitStats, boolean autofit, double unitsPerPixel) { + double df=fitStats[2], sse=fitStats[3], sigma=sigma(fitStats[6],unitsPerPixel); + double chi=Double.NaN, reduced=Double.NaN, q=Double.NaN; + if(autofit && df>0 && CurveFitReport.isFinite(sse) && sse>=0 && positive(sigma)) { + chi=mode==ESTIMATED?df:sse/sigma/sigma; + reduced=chi/df; + if(mode!=ESTIMATED)q=survival(chi,df); + } + return new double[]{sigma,chi,reduced,q}; + } + /** Q(df/2, chi-square/2), the regularized upper incomplete gamma. + * NIST DLMF 8.2.4; lower series for x=parameterBottom.y+parameterBottom.height,"parameter information above statistics"); check(fitter.getVisibleRect().contains(statsBounds),"all statistics visible in parent fit area"); check(stats.getHeight()==stats.getPreferredSize().height,"statistics remain compact"); + java.lang.reflect.Field uf=DatasetCurveFitter.class.getDeclaredField("uncertaintyPanel");uf.setAccessible(true);JPanel uncertainty=(JPanel)uf.get(fitter); + Rectangle uncertaintyBounds=SwingUtilities.convertRectangle(uncertainty,new Rectangle(0,0,uncertainty.getWidth(),uncertainty.getHeight()),fitter); + check(fitter.getVisibleRect().contains(uncertaintyBounds),"uncertainty controls visible in fit area"); check(tab.splitPanes[1].getTopComponent().getHeight()>100,"plot retains usable height"); check(Math.abs(fitter.getHeight()-fitter.prepareFitLayout(fitter.getWidth()))<=4,"fit area stays at required height"); if(args.length>0){BufferedImage image=new BufferedImage(tool.getContentPane().getWidth(),tool.getContentPane().getHeight(),BufferedImage.TYPE_INT_RGB);Graphics2D g=image.createGraphics();tool.getContentPane().printAll(g);g.dispose();javax.imageio.ImageIO.write(image,"png",new java.io.File(args[0]+"/data-tool-"+size[0]+"-font"+level+".png"));} diff --git a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java new file mode 100644 index 00000000..8f04d53f --- /dev/null +++ b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java @@ -0,0 +1,140 @@ +package org.opensourcephysics.tools; + +import org.opensourcephysics.display.*; +import org.opensourcephysics.media.core.VideoIO; +import javax.swing.SwingUtilities; + +/** Scientific and clipboard regression checks, also transpiled for SwingJS. */ +public class CurveFitPhysicsTest { + static int passed; + static void check(boolean ok,String name) { if(!ok)throw new AssertionError(name);passed++; } + static void near(double a,double b,String name) { check(Math.abs(a-b)<1e-7*Math.max(1,Math.abs(b)),name+": "+a+" != "+b); } + public static void main(String[] args) throws Exception { + try { if(OSPRuntime.isJS)run();else SwingUtilities.invokeAndWait(()->run()); } + catch(Throwable t) { t.printStackTrace();if(!OSPRuntime.isJS)System.exit(1);throw new RuntimeException(t); } + System.out.println("Passed: "+passed); + if(!OSPRuntime.isJS)System.exit(0); + } + static void run() { + OSPRuntime.setPreferredDecimalSeparator("."); + DatasetManager manager=new DatasetManager();manager.setXPointsLinked(true); + manager.setXYColumnNames(0,"t","x");manager.append(0,new double[]{0,1,2,3},new double[]{1.1,2.9,4.9,7.1}); + manager.setXYColumnNames(1,"t","v_{x}");manager.append(1,new double[]{0,1,2,3},new double[]{2,2,2,2}); + DataTable table=new DataTable();table.add(manager.model); + table.refreshTable(DataTable.MODE_CREATE); + String delimiter=VideoIO.getDelimiter(); + for(String delim:new String[]{"\t",","}) { + VideoIO.setDelimiter(delim); + for(boolean formatted:new boolean[]{false,true}) { + table.setUnits("t",null,null);table.setUnits("x",null,null);table.setUnits("v_{x}",null,null); + String bare=table.getData(formatted).toString(); + check(!bare.substring(0,bare.indexOf('\n')).contains("("),"no units"); + check(bare.startsWith("t"+delim+"x"+delim+"vx"),"unitless headers preserve legacy subscript normalization"); + table.setUnits("t"," s",null);table.setUnits("x"," m",null);table.setUnits("v_{x}"," m/s²",null); + String full=table.getData(formatted).toString(); + check(full.startsWith("t (s)"+delim+"x (m)"+delim+"v_x (m/s^2)"),"all unit headers "+delim+formatted); + check(full.substring(full.indexOf('\n')).equals(bare.substring(bare.indexOf('\n'))),"numeric cells unchanged "+formatted); + table.setUnits("x",null,null); + check(table.getData(formatted).toString().startsWith("t (s)"+delim+"x"+delim),"mixed headers"); + DatasetManager[] pasted=DataTool.parseData(full,null); + check(pasted!=null,"export parses"); + DataTable again=new DataTable();again.add(pasted[0].model);again.refreshTable(DataTable.MODE_CREATE); + String twice=again.getData(false).toString(); + check(!twice.contains("(m) (m)") && twice.contains("x (m)"),"round trip no duplicate"); + } + } + VideoIO.setDelimiter(delimiter); + check(ExportText.header("x (m)","m").equals("x (m)"),"known suffix not doubled"); + check(ExportText.ascii("A*t^{2} + B*t + C").equals("A*t^2 + B*t + C"),"plain equation"); + for(int degree=0;degree<=3;degree++) { + KnownPolynomial polynomial=new KnownPolynomial(new double[degree+1]); + for(int i=0;i<=degree;i++) { + int power=degree-i;polynomial.setParameterValue(i,7); + String expected=power==0?"m":power==1?"m/s":"m/s^"+power; + check(CurveFitReport.coefficientUnits(polynomial,i,"s","m").equals(expected),"polynomial degree "+degree+" term "+power); + check(CurveFitReport.coefficientUnits(polynomial,i,null,"m").equals(power==0?"m":""),"unknown x unit"); + check(CurveFitReport.coefficientUnits(polynomial,i,"s",null).isEmpty(),"unknown y unit"); + } + } + UserFunction custom=new UserFunction("custom");custom.setParameters(new String[]{"A"},new double[]{1},null);custom.setExpression("A*x",new String[]{"x"}); + check(CurveFitReport.coefficientUnits(custom,0,"s","m").isEmpty(),"user function units not guessed"); + Dataset data=CurveFitConstraintTest.data(new double[]{0,1,2,3},new double[]{1.1,2.9,4.9,7.1}); + DatasetCurveFitter fitter=CurveFitConstraintTest.line(data); + near(fitter.fit.getParameterValue(0),2,"default slope");near(fitter.fit.getParameterValue(1),1,"default intercept"); + double[] stats=CurveFitReport.statistics(fitter.fit,data,null,true); + FitUncertainty estimated=FitUncertainty.estimated();double[] chi=estimated.statistics(stats,true,Double.NaN); + near(chi[0],Math.sqrt(.02),"estimated sigma");near(chi[1],2,"default chi df");near(chi[2],1,"default reduced chi");check(Double.isNaN(chi[3]),"default Q unavailable"); + double oldError=fitter.getUncertainty(0),slope=fitter.fit.getParameterValue(0),intercept=fitter.fit.getParameterValue(1); + fitter.setUncertaintyModel(FitUncertainty.CONSTANT,.2); + check(fitter.fit.getParameterValue(0)==slope && fitter.fit.getParameterValue(1)==intercept,"constant sigma leaves coefficients bitwise unchanged"); + near(fitter.getUncertainty(0),oldError*.2/Math.sqrt(.02),"profile uncertainty uses supplied scale"); + chi=fitter.getUncertaintyModel().statistics(stats,true,Double.NaN); + near(chi[0],.2,"supplied sigma not rescaled");near(chi[1],1,"specified chi");near(chi[2],.5,"specified reduced chi");near(chi[3],Math.exp(-.5),"Q df2"); + near(FitUncertainty.survival(3.841458820694124,1),.05,"Q df1"); + near(FitUncertainty.survival(18.307038053275146,10),.05,"Q df10"); + near(FitUncertainty.survival(0,3),1,"Q zero");near(FitUncertainty.survival(1000,2),Math.exp(-500),"Q extreme"); + fitter.setUncertaintyModel(FitUncertainty.CONSTANT,Double.NaN); + check(Double.isNaN(fitter.getUncertainty(0)),"invalid supplied sigma no estimated fallback"); + chi=fitter.getUncertaintyModel().statistics(stats,true,Double.NaN);check(Double.isNaN(chi[1]),"invalid chi unavailable"); + fitter.setAutoFit(false);fitter.setUncertaintyModel(FitUncertainty.CONSTANT,.2); + check(!fitter.isAutoFit() && fitter.fit.getParameterValue(0)==slope && Double.isNaN(fitter.getUncertainty(0)),"manual behavior retained"); + for(double pixels:new double[]{.5,1,1.5,2,2.5,3,.73}) { + FitUncertainty model=new FitUncertainty(FitUncertainty.PIXELS,pixels); + near(model.sigma(Double.NaN,.01),pixels*.01,"fractional pixel conversion"); + double[] result=model.statistics(stats,true,.01); + near(result[1],stats[3]/Math.pow(pixels*.01,2),"pixel chi"); + check(Double.isNaN(model.sigma(1,Double.NaN)),"derived quantity has no pixel conversion"); + } + UserFunction redundant=new UserFunction("RedundantPhysics");redundant.setParameters(new String[]{"A","B"},new double[]{1,1},null);redundant.setExpression("(A+B)*x",new String[]{"x"}); + stats=CurveFitReport.statistics(redundant,data,null,true);near(stats[1],2,"nominal free count");near(stats[8],1,"rank one");near(stats[2],3,"rank df"); + chi=estimated.statistics(stats,true,Double.NaN);near(chi[1],3,"rank default chi"); + stats=CurveFitReport.statistics(fitter.fit,data,new boolean[]{true,false},true);near(stats[8],1,"fixed rank");near(stats[2],3,"fixed df"); + Dataset exact=CurveFitConstraintTest.data(new double[]{0,1,2,3},new double[]{1,3,5,7}); + fitter=CurveFitConstraintTest.line(exact);stats=CurveFitReport.statistics(fitter.fit,exact,null,true); + chi=estimated.statistics(stats,true,Double.NaN);near(chi[0],0,"perfect residual sigma zero");check(Double.isNaN(chi[1]),"perfect estimated chi undefined"); + fitter.setUncertaintyModel(FitUncertainty.CONSTANT,.2);check(fitter.getUncertainty(0)>0,"perfect fit known noise positive profile error"); + chi=fitter.getUncertaintyModel().statistics(stats,true,Double.NaN);near(chi[1],0,"perfect specified chi zero");near(chi[3],1,"perfect specified Q one"); + // Existing UserFunction profile-curvature path, including its scale. + fitter=CurveFitConstraintTest.line(data); + UserFunction exponential=new UserFunction("PhysicsExp"); + exponential.setParameters(new String[]{"A"},new double[]{.4},null); + exponential.setExpression("exp(A*x)",new String[]{"x"}); + fitter.fit=exponential;fitter.fit(exponential,true); + double originalA=fitter.fit.getParameterValue(0),profile=fitter.getUncertainty(0); + double residualSigma=CurveFitReport.statistics(fitter.fit,data,null,true)[6]; + fitter.setUncertaintyModel(FitUncertainty.CONSTANT,residualSigma*2); + check(fitter.fit.getParameterValue(0)==originalA,"user function coefficient unchanged by scale"); + check(profile>0 && Math.abs(fitter.getUncertainty(0)/profile-2)<.02,"user function profile errors scale without covariance replacement"); + fitter.fit=redundant;fitter.fit(redundant,true); + check(Double.isNaN(fitter.getUncertainty(0)) && Double.isNaN(fitter.getUncertainty(1)),"redundant supplied-sigma profile errors remain unavailable"); + + DataTool tool=new DataTool(manager);DataToolTab tab=tool.getTab(0);tab.checkGUI(); + final double[] scale={.01}; + tab.setFitMetadataProvider(new FitMetadataProvider() { + public String getUnits(String column) { return column.equals("t")?"s":column.equals("x")?"m":null; } + public double getYUnitsPerPixel(String column) { return column.equals("x")?scale[0]:Double.NaN; } + }); + tab.setWorkingColumns("t","x"); + fitter=tab.getCurveFitter();fitter.selectFit(fitter.getPolyFitNameOfDegree(1));fitter.setActiveAndFit(true); + double measuredSlope=fitter.fit.getParameterValue(0); + near(fitter.getYUnitsPerPixel(),.01,"host position calibration reaches fitter: "+fitter.getData().getYColumnName()); + for(double pixels:new double[]{.5,1,1.5,2,2.5,3,.73}) { + fitter.setUncertaintyModel(FitUncertainty.PIXELS,pixels); + near(fitter.getUncertaintyModel().sigma(Double.NaN,fitter.getYUnitsPerPixel()),pixels*.01,"host fractional pixel scale"); + check(fitter.fit.getParameterValue(0)==measuredSlope,"pixel choices preserve fitted coefficient"); + } + scale[0]=.02;fitter.refreshUncertaintyModel();near(fitter.getYUnitsPerPixel(),.02,"live calibration refresh"); + tab.setWorkingColumns("t","v_{x}"); + check(Double.isNaN(fitter.getYUnitsPerPixel()),"derived variable does not inherit position calibration"); + fitter.setUncertaintyModel(FitUncertainty.PIXELS,1); + check(Double.isNaN(fitter.getUncertainty(0)),"ineligible pixel sigma not applied"); + tool.dispose(); + fitter=CurveFitConstraintTest.line(exact); + String report=CurveFitReport.create(fitter.fit,exact,null,new double[]{.1,.1},true,true,"s","m",estimated,Double.NaN); + check(report.contains("X variable\tt (s)") && report.contains("Y variable\ty (m)"),"fit variable units"); + check(report.contains("Parameter\tCoefficients\tStandard Error\tUnits\tFixed"),"coefficient units column"); + check(report.contains("R Square (centered)"),"centered R2 retained"); + report=CurveFitReport.create(redundant,data,null,null,true,true,"s","m",estimated,Double.NaN); + check(report.contains("not independently identifiable"),"rank warning exposed"); + } +} diff --git a/test/org/opensourcephysics/tools/CurveFitReportTest.java b/test/org/opensourcephysics/tools/CurveFitReportTest.java index 8cd130f5..ded96714 100644 --- a/test/org/opensourcephysics/tools/CurveFitReportTest.java +++ b/test/org/opensourcephysics/tools/CurveFitReportTest.java @@ -24,11 +24,11 @@ public static void main(String[] args) throws Exception { close(stats[7], 20.04, "total sum of squares"); String report = CurveFitReport.create(line, data, null, new double[] {.0632455532033676, .1183215956619923}, true, true); String[] rows = report.split("\n"); - check(rows[0].startsWith("SUMMARY OUTPUT\tposition vs time\t"), "simple title and data-name fallback"); - check(rows[1].contains("position = ") && rows[1].contains("Automatic fit"), "second header contains model, equation, and mode"); + check(rows[0].startsWith("FIT\t"), "simple title and data-name fallback"); + check(report.contains("Equation\tposition = ") && rows[1].contains("Automatic fit"), "second header contains model, equation, and mode"); for (String row : rows) check(splitTabs(row).length == 5, "five aligned spreadsheet columns"); - check(report.indexOf("Regression Statistics") < report.indexOf("ANOVA") - && report.indexOf("ANOVA") < report.indexOf("Parameter\tCoefficients"), "Excel-style section order"); + check(report.indexOf("GOODNESS OF FIT") < report.indexOf("ANOVA") + && report.indexOf("ANOVA") < report.indexOf("Parameter\tCoefficients"), "physics sections before supplementary ANOVA"); close(cell(report, "Multiple R", 1), Math.sqrt(stats[5]), "Multiple R"); close(cell(report, "Adjusted R Square (centered)", 1), 1 - (.04/2)/(20.04/3), "Adjusted R Square"); close(cell(report, "Regression", 1), 1, "regression df"); @@ -45,11 +45,11 @@ public static void main(String[] args) throws Exception { check(rows[rows.length - 1].startsWith("B\t"), "report ends with coefficients, without a prose footer"); report = CurveFitReport.create(line, data, new boolean[] {true, false}, new double[] {.05, .01}, true, true); close(cell(report, "Residual", 1), 3, "fixed parameter excluded from p"); - check(report.contains("A\t2.0\tN/A\tYes"), "fixed coefficient has no uncertainty"); + check(report.contains("A\t2.0\tN/A\t\tYes"), "fixed coefficient has no uncertainty"); check(report.contains("Regression\tN/A\tN/A\tN/A\tN/A"), "no classical regression ANOVA for constrained fits"); report = CurveFitReport.create(line, data, null, new double[] {.05, .01}, false, true); - check(report.contains("Manual parameters") && report.contains("A\t2.0\tN/A\tNo"), "manual mode does not reuse uncertainties"); - check(report.contains("Standard Error\tN/A"), "manual mode has no fitted residual standard error"); + check(report.contains("Manual parameters") && report.contains("A\t2.0\tN/A\t\tNo"), "manual mode does not reuse uncertainties"); + check(report.contains("Residual standard error\tN/A"), "manual mode has no fitted residual standard error"); KnownPolynomial parabola = new KnownPolynomial(new double[] {0, 0, 1}); Dataset curved = new Dataset(); curved.append(new double[] {-2, -1, 0, 1, 2}, new double[] {4, 1, 0, 1, 4}); @@ -106,7 +106,7 @@ public static void main(String[] args) throws Exception { check(Double.doubleToLongBits(Double.parseDouble(first[1])) == Double.doubleToLongBits(value), "coefficient round-trips exactly"); check(Double.doubleToLongBits(Double.parseDouble(first[2])) == Double.doubleToLongBits(sigma), "uncertainty round-trips exactly"); check(params.split("\n").length == 3, "parameters-only export has one header and two rows"); - check(first[3].equals("No") && first[4].isEmpty(), "coefficient and uncertainty each exported once"); + check(first[4].equals("No") && first[3].isEmpty(), "coefficient and uncertainty each exported once"); check(line.getParameterValue(0) == value, "copying leaves model unchanged"); OSPRuntime.setPreferredDecimalSeparator(","); params = CurveFitReport.create(line, data, null, new double[] {sigma,.01}, true, false); From fbdcb6e343d7207d692fb746951bd28b6a280603 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Sun, 6 Sep 2026 20:56:37 -0400 Subject: [PATCH 08/21] Default to a compact fit report with optional full analysis --- .../resources/tools/tools.properties | 5 ++- .../tools/CurveFitReport.java | 35 +++++++++++++++++++ .../tools/DatasetCurveFitter.java | 35 ++++++++++++++++--- test/fit-report-physics.md | 4 +++ .../tools/CurveFitPhysicsTest.java | 2 ++ .../tools/CurveFitReportTest.java | 12 +++++++ 6 files changed, 87 insertions(+), 6 deletions(-) diff --git a/src/org/opensourcephysics/resources/tools/tools.properties b/src/org/opensourcephysics/resources/tools/tools.properties index cad384a1..ef3d5a54 100644 --- a/src/org/opensourcephysics/resources/tools/tools.properties +++ b/src/org/opensourcephysics/resources/tools/tools.properties @@ -909,7 +909,7 @@ LibraryTreePanel.Dialog.Open.Title=Open Resource? # Curve fit clipboard report DatasetCurveFitter.Button.CopyFitReport=Copy Fit Report -DatasetCurveFitter.Button.CopyFitReport.Tooltip=Copy a spreadsheet-ready regression summary, ANOVA, and coefficients +DatasetCurveFitter.Button.CopyFitReport.Tooltip=Copy parameters and the fit statistics shown on screen DatasetCurveFitter.Report.Title=Curve fit report DatasetCurveFitter.Report.Data=Data DatasetCurveFitter.Report.Model=Model @@ -987,3 +987,6 @@ DatasetCurveFitter.Report.Parameters=PARAMETERS DatasetCurveFitter.Report.Units=Units DatasetCurveFitter.Report.ResidualStandardError=Residual standard error + +DatasetCurveFitter.Menuitem.CopyFullReport=Copy Full Fit Report +DatasetCurveFitter.Button.CopyOptions.Tooltip=More copy options (full fit analysis) diff --git a/src/org/opensourcephysics/tools/CurveFitReport.java b/src/org/opensourcephysics/tools/CurveFitReport.java index 8eb21aa6..1e67b0eb 100644 --- a/src/org/opensourcephysics/tools/CurveFitReport.java +++ b/src/org/opensourcephysics/tools/CurveFitReport.java @@ -78,6 +78,41 @@ static String create(KnownFunction fit, Dataset data, boolean[] fixed, } return out.toString(); } + /** Compact classroom report; numeric cells retain the full stored precision. */ + static String createSummary(KnownFunction fit, Dataset data, boolean[] fixed, + double[] uncertainties, boolean autofit, String xUnits, String yUnits, + FitUncertainty uncertainty, double unitsPerPixel) { + StringBuilder out = new StringBuilder(); + String x = data == null ? "x" : data.getXColumnName(); + String y = data == null ? "y" : data.getYColumnName(); + row(out, label("Title"), data == null ? "" : data.getName()); + row(out, label("Model"), fit.getName(), label(autofit ? "Auto" : "Manual")); + row(out, label("Equation"), org.opensourcephysics.display.ExportText.ascii(y + " = " + fit.getExpression(x))); + row(out, label("XVariable"), org.opensourcephysics.display.ExportText.header(x, xUnits)); + row(out, label("YVariable"), org.opensourcephysics.display.ExportText.header(y, yUnits)); + row(out); + // Reuse the coefficient-only export to preserve units, fixed/manual behavior, + // and uncertainty values in both report formats. + out.append(create(fit, data, fixed, uncertainties, autofit, false, + xUnits, yUnits, uncertainty, unitsPerPixel)); + row(out); + double[] stats = statistics(fit, data, fixed, autofit); + for (int i = 0; i < STATISTIC_KEYS.length; i++) { + String key = STATISTIC_KEYS[i]; + String title = i == 4 ? label("RMSResidual") + : ToolsRes.getString("DatasetCurveFitter.Statistics." + key); + row(out, title, i < 3 ? integer(stats[i]) : number(stats[i])); + } + if (autofit && stats[8] >= 0 && stats[8] < stats[1]) row(out, label("Identifiability")); + if (uncertainty.mode != FitUncertainty.ESTIMATED) { + row(out, label("SpecifiedSigma"), number(uncertainty.statistics(stats, autofit, unitsPerPixel)[0]), + org.opensourcephysics.display.ExportText.ascii(yUnits)); + if (uncertainty.mode == FitUncertainty.PIXELS) + row(out, label("SigmaPixels"), number(uncertainty.value), "pixels"); + } + return out.toString(); + } + static String coefficientUnits(KnownFunction fit,int parameter,String xUnits,String yUnits) { if(!(fit instanceof KnownPolynomial))return ""; String x=org.opensourcephysics.display.ExportText.ascii(xUnits).trim(); diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index 3249be95..9fdf3157 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -306,7 +306,7 @@ public void setAutoFit(boolean autofit) { // GUI - private JButton colorButton, closeButton, copyFitReportButton; + private JButton colorButton, closeButton, copyFitReportButton, copyFitOptionsButton; private JCheckBox autofitCheckBox; private JLabel fitLabel, eqnLabel, rmsLabel; private JToolBar fitBar, eqnBar, rmsBar; @@ -645,6 +645,10 @@ private void refreshFitStatistics() { /** Copies a snapshot of the current fit without fitting or rounding its data. */ private void copyFitResults(boolean includeStatistics) { + copyFitResults(includeStatistics, false); + } + + private void copyFitResults(boolean includeStatistics, boolean fullReport) { if (paramTable.isEditing() && !paramTable.getCellEditor().stopCellEditing()) return; if (fit == null) @@ -655,7 +659,10 @@ private void copyFitResults(boolean includeStatistics) { double[] sigma = new double[fit.getParameterCount()]; for (int i = 0; i < sigma.length; i++) sigma[i] = getUncertainty(i); - OSPRuntime.copy(CurveFitReport.create(fit, dataset, fixedParams.get(fit), sigma, + OSPRuntime.copy(includeStatistics && !fullReport + ? CurveFitReport.createSummary(fit, dataset, fixedParams.get(fit), sigma, + autofit, variableUnits(true), variableUnits(false), uncertaintyModel, getYUnitsPerPixel()) + : CurveFitReport.create(fit, dataset, fixedParams.get(fit), sigma, autofit, includeStatistics, variableUnits(true), variableUnits(false), uncertaintyModel, getYUnitsPerPixel()), null); } @@ -1132,7 +1139,20 @@ public void layoutContainer(Container target) { rmsBar.addSeparator(); copyFitReportButton = new JButton(); copyFitReportButton.addActionListener(e -> copyFitResults(true)); - rmsBar.add(copyFitReportButton); + JPanel copyButtons = new JPanel(new BorderLayout()); + copyButtons.add(copyFitReportButton, BorderLayout.CENTER); + copyFitOptionsButton = new JButton("..."); + copyFitOptionsButton.setMargin(new java.awt.Insets(2, 4, 2, 4)); + copyFitOptionsButton.addActionListener(e -> { + JPopupMenu menu = new JPopupMenu(); + JMenuItem full = new JMenuItem(ToolsRes.getString("DatasetCurveFitter.Menuitem.CopyFullReport")); + full.addActionListener(ev -> copyFitResults(true, true)); + menu.add(full); + FontSizer.setFonts(menu); + menu.show(copyFitOptionsButton, 0, copyFitOptionsButton.getHeight()); + }); + copyButtons.add(copyFitOptionsButton, BorderLayout.EAST); + rmsBar.add(copyButtons); rmsPanel.add(rmsBar, BorderLayout.NORTH); statisticsPanel = new JPanel(new java.awt.GridLayout(0, 2, 10, 3)); statisticsPanel.setBorder(BorderFactory.createEmptyBorder(4, 4, 4, 4)); @@ -1234,7 +1254,9 @@ protected void processPropertyChange(PropertyChangeEvent e) { * Refreshes the GUI. */ protected void refreshGUI() { - copyFitReportButton.setText(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport")); + copyFitOptionsButton.setToolTipText(ToolsRes.getString("DatasetCurveFitter.Button.CopyOptions.Tooltip")); + if (!OSPRuntime.isJS) copyFitOptionsButton.getAccessibleContext().setAccessibleName(ToolsRes.getString("DatasetCurveFitter.Button.CopyOptions.Tooltip")); + copyFitReportButton.setText(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport")); copyFitReportButton.setToolTipText(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport.Tooltip")); autofitCheckBox.setText(ToolsRes.getString("Checkbox.Autofit.Label")); //$NON-NLS-1$ rmsLabel.setText(ToolsRes.getString("DatasetCurveFitter.Label.RMSDeviation")); //$NON-NLS-1$ @@ -2153,7 +2175,10 @@ public void showPopup(MouseEvent e) { item = new JMenuItem(ToolsRes.getString("DatasetCurveFitter.Button.CopyFitReport")); item.addActionListener(ev -> copyFitResults(true)); popup.add(item); - popup.addSeparator(); + item = new JMenuItem(ToolsRes.getString("DatasetCurveFitter.Menuitem.CopyFullReport")); + item.addActionListener(ev -> copyFitResults(true, true)); + popup.add(item); + popup.addSeparator(); JCheckBoxMenuItem scientificNotationItem = new JCheckBoxMenuItem("Scientific notation"); //$NON-NLS-1$ scientificNotationItem.setSelected(!isFixedDecimalFormat); spinCellEditor.field.applyDefaultPattern(!isFixedDecimalFormat); //$NON-NLS-1$ diff --git a/test/fit-report-physics.md b/test/fit-report-physics.md index e170681b..c846c267 100644 --- a/test/fit-report-physics.md +++ b/test/fit-report-physics.md @@ -86,3 +86,7 @@ Firefox and WebKit (290 checks per browser). Browser interaction checks exercise selection, resizing, fixed parameters, supplied sigma, and copying the report. These local runs cover macOS and browser engines; they do not constitute new native Windows/Linux runtime testing. + +### Compact and full copied reports + +Copy Fit Report exports the parameter table and the statistics shown on screen, with model, equation, variable units, and full-precision numeric cells. It omits ANOVA and advanced goodness-of-fit analysis. The adjacent `...` menu offers **Copy Full Fit Report**, also available in the parameter context menu. Both formats preserve manual/fixed parameter semantics and uncertainty units. Specified measurement uncertainty and identifiability warnings remain visible in the compact report when applicable. diff --git a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java index 8f04d53f..00d5345b 100644 --- a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java +++ b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java @@ -128,6 +128,8 @@ static void run() { check(Double.isNaN(fitter.getYUnitsPerPixel()),"derived variable does not inherit position calibration"); fitter.setUncertaintyModel(FitUncertainty.PIXELS,1); check(Double.isNaN(fitter.getUncertainty(0)),"ineligible pixel sigma not applied"); + // This fixture is disposable; do not open a modal save prompt during cleanup. + tab.tabChanged(false); tool.dispose(); fitter=CurveFitConstraintTest.line(exact); String report=CurveFitReport.create(fitter.fit,exact,null,new double[]{.1,.1},true,true,"s","m",estimated,Double.NaN); diff --git a/test/org/opensourcephysics/tools/CurveFitReportTest.java b/test/org/opensourcephysics/tools/CurveFitReportTest.java index ded96714..9234431f 100644 --- a/test/org/opensourcephysics/tools/CurveFitReportTest.java +++ b/test/org/opensourcephysics/tools/CurveFitReportTest.java @@ -23,6 +23,18 @@ public static void main(String[] args) throws Exception { close(stats[6], Math.sqrt(.02), "standard error uses n-p"); close(stats[7], 20.04, "total sum of squares"); String report = CurveFitReport.create(line, data, null, new double[] {.0632455532033676, .1183215956619923}, true, true); + String summary = CurveFitReport.createSummary(line, data, null, new double[] {.05, .01}, true, + "s", "m", FitUncertainty.estimated(), Double.NaN); + check(summary.contains("X variable\ttime (s)") && summary.contains("Y variable\tposition (m)"), "summary variable units"); + check(summary.contains("A\t2.0\t0.05\tm/s\tNo"), "summary full precision coefficients and units"); + check(!summary.contains("ANOVA") && !summary.contains("Chi square") && !summary.contains("Adjusted R"), "summary omits advanced analysis"); + check(summary.contains("Points\t4") && summary.contains("Degrees of freedom\t2"), "summary screen counts"); + close(cell(summary, "SSE", 1), stats[3], "summary SSE unchanged"); + close(cell(summary, "R-squared", 1), stats[5], "summary centered R squared unchanged"); + for (String row : summary.split("\n")) check(splitTabs(row).length == 5, "summary aligned spreadsheet columns"); + String manualSummary = CurveFitReport.createSummary(line, data, null, new double[] {.05, .01}, false, + null, null, FitUncertainty.estimated(), Double.NaN); + check(manualSummary.contains("A\t2.0\tN/A") && manualSummary.contains("Degrees of freedom\tN/A"), "summary manual behavior"); String[] rows = report.split("\n"); check(rows[0].startsWith("FIT\t"), "simple title and data-name fallback"); check(report.contains("Equation\tposition = ") && rows[1].contains("Automatic fit"), "second header contains model, equation, and mode"); From 1d6dde1ce6275e93523ba3fd4f10c9e5d92a2c36 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Sun, 6 Sep 2026 22:22:48 -0400 Subject: [PATCH 09/21] Clarify uncertainty choices and preserve scale when switching units --- .../resources/tools/tools.properties | 8 +-- .../tools/DatasetCurveFitter.java | 52 ++++++++++++++++--- test/fit-report-physics.md | 2 + .../tools/CurveFitDataToolLayoutTest.java | 7 +++ .../tools/CurveFitPhysicsTest.java | 11 ++++ 5 files changed, 69 insertions(+), 11 deletions(-) diff --git a/src/org/opensourcephysics/resources/tools/tools.properties b/src/org/opensourcephysics/resources/tools/tools.properties index ef3d5a54..cfe91f06 100644 --- a/src/org/opensourcephysics/resources/tools/tools.properties +++ b/src/org/opensourcephysics/resources/tools/tools.properties @@ -957,9 +957,9 @@ DatasetCurveFitter.Report.Total=Total DatasetCurveFitter.Report.Coefficients=Coefficients DatasetCurveFitter.Uncertainty.Label=Data uncertainty: -DatasetCurveFitter.Uncertainty.Estimated=Estimated from residuals -DatasetCurveFitter.Uncertainty.Constant=Constant sigma_y -DatasetCurveFitter.Uncertainty.Pixels=Position sigma (pixels) +DatasetCurveFitter.Uncertainty.Estimated=Unweighted - estimate from residuals +DatasetCurveFitter.Uncertainty.Constant=Known uncertainty (data units) +DatasetCurveFitter.Uncertainty.Pixels=Known uncertainty (pixels) DatasetCurveFitter.Uncertainty.Custom=Choose a value or type a custom positive uncertainty. DatasetCurveFitter.Uncertainty.Invalid=Unavailable: enter a positive value and check calibration. DatasetCurveFitter.Report.Fit=FIT @@ -990,3 +990,5 @@ DatasetCurveFitter.Report.ResidualStandardError=Residual standard error DatasetCurveFitter.Menuitem.CopyFullReport=Copy Full Fit Report DatasetCurveFitter.Button.CopyOptions.Tooltip=More copy options (full fit analysis) + +DatasetCurveFitter.Uncertainty.Value=Uncertainty value: diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index 9fdf3157..1052f3d4 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -206,6 +206,25 @@ public void setUncertaintyModel(int mode,double value) { uncertaintyModel=new FitUncertainty(mode,value); refreshUncertaintyModel(); } + /** Choose a starting scale without interpreting a physical-unit number as pixels. */ + void selectUncertaintyMode(int mode) { + double value = uncertaintyModel.value; + double calibration = getYUnitsPerPixel(); + if (mode != uncertaintyModel.mode) { + if (mode == FitUncertainty.PIXELS) { + value = uncertaintyModel.mode == FitUncertainty.CONSTANT && FitUncertainty.positive(calibration) + ? value / calibration : 1.0; + } else if (mode == FitUncertainty.CONSTANT) { + if (uncertaintyModel.mode == FitUncertainty.PIXELS) value *= calibration; + else if (FitUncertainty.positive(calibration)) value = calibration; + else value = fit == null ? Double.NaN + : CurveFitReport.statistics(fit, dataset, fixedParams.get(fit), autofit)[6]; + if (!FitUncertainty.positive(value)) value = Double.NaN; + } + } + setUncertaintyModel(mode, value); + } + private String variableUnits(boolean x) { return tab==null || dataset==null ? null : tab.dataTable.getUnits(x?dataset.getXColumnName():dataset.getYColumnName()); } @@ -237,10 +256,16 @@ private void refreshUncertaintyControls() { uncertaintyChoice.removeItemAt(2); uncertaintyChoice.setSelectedIndex(uncertaintyModel.mode); uncertaintyValue.setEnabled(uncertaintyModel.mode!=FitUncertainty.ESTIMATED); - if(uncertaintyModel.mode!=FitUncertainty.ESTIMATED)uncertaintyValue.setSelectedItem(Double.toString(uncertaintyModel.value)); + uncertaintyValue.removeAllItems(); + if (uncertaintyModel.mode == FitUncertainty.PIXELS) + for (String preset : new String[]{"0.5", "1.0", "1.5", "2.0", "2.5", "3.0"}) uncertaintyValue.addItem(preset); + if(uncertaintyModel.mode!=FitUncertainty.ESTIMATED)uncertaintyValue.setSelectedItem(FitUncertainty.positive(uncertaintyModel.value) + ? Double.toString(uncertaintyModel.value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : ""); + else uncertaintyValue.setSelectedItem(""); double sigma=uncertaintyModel.sigma(Double.NaN,getYUnitsPerPixel()); uncertaintyStatus.setText(uncertaintyModel.mode==FitUncertainty.ESTIMATED?"": - FitUncertainty.positive(sigma)?org.opensourcephysics.display.ExportText.ascii(variableUnits(false)): + FitUncertainty.positive(sigma)?(uncertaintyModel.mode == FitUncertainty.PIXELS ? "pixels" + : org.opensourcephysics.display.ExportText.ascii(variableUnits(false))): ToolsRes.getString("DatasetCurveFitter.Uncertainty.Invalid")); updatingUncertaintyControls=false; } @@ -1163,11 +1188,16 @@ public void layoutContainer(Container target) { } JPanel statisticsContainer = new JPanel(new BorderLayout()); statisticsContainer.add(statisticsPanel, BorderLayout.NORTH); - uncertaintyPanel=new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT,4,2)); - uncertaintyPanel.add(new JLabel(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Label"))); + uncertaintyPanel=new JPanel(new java.awt.GridLayout(0,1,0,2)); + JPanel uncertaintyModeRow = new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT,4,2)); + JPanel uncertaintyValueRow = new JPanel(new java.awt.FlowLayout(java.awt.FlowLayout.LEFT,4,2)); + uncertaintyModeRow.add(new JLabel(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Label"))); uncertaintyChoice=new JComboBox(new String[]{ToolsRes.getString("DatasetCurveFitter.Uncertainty.Estimated"),ToolsRes.getString("DatasetCurveFitter.Uncertainty.Constant")}); - uncertaintyValue=new JComboBox(new String[]{"0.5","1.0","1.5","2.0","2.5","3.0"}); - uncertaintyValue.setEditable(true);uncertaintyValue.setSelectedItem("1.0"); + uncertaintyValue=new JComboBox(); + uncertaintyValue.setPrototypeDisplayValue("0.0000000000"); + uncertaintyValue.setEditable(true); + if (uncertaintyValue.getEditor().getEditorComponent() instanceof JTextField) + ((JTextField) uncertaintyValue.getEditor().getEditorComponent()).setColumns(12); uncertaintyValue.setToolTipText(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Custom")); uncertaintyStatus=new JLabel(); java.awt.event.ActionListener changed=e->{ @@ -1177,8 +1207,14 @@ public void layoutContainer(Container target) { catch(Exception ex) { /* Keep invalid supplied input unavailable, never estimate it. */ } setUncertaintyModel(uncertaintyChoice.getSelectedIndex(),value); }; - uncertaintyChoice.addActionListener(changed);uncertaintyValue.addActionListener(changed); - uncertaintyPanel.add(uncertaintyChoice);uncertaintyPanel.add(uncertaintyValue);uncertaintyPanel.add(uncertaintyStatus); + uncertaintyChoice.addActionListener(e -> { + if (!updatingUncertaintyControls) selectUncertaintyMode(uncertaintyChoice.getSelectedIndex()); + }); + uncertaintyValue.addActionListener(changed); + uncertaintyModeRow.add(uncertaintyChoice); + uncertaintyValueRow.add(new JLabel(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Value"))); + uncertaintyValueRow.add(uncertaintyValue);uncertaintyValueRow.add(uncertaintyStatus); + uncertaintyPanel.add(uncertaintyModeRow);uncertaintyPanel.add(uncertaintyValueRow); statisticsContainer.add(uncertaintyPanel,BorderLayout.SOUTH); refreshUncertaintyControls(); add(statisticsContainer, BorderLayout.CENTER); diff --git a/test/fit-report-physics.md b/test/fit-report-physics.md index c846c267..c5417c8b 100644 --- a/test/fit-report-physics.md +++ b/test/fit-report-physics.md @@ -90,3 +90,5 @@ native Windows/Linux runtime testing. ### Compact and full copied reports Copy Fit Report exports the parameter table and the statistics shown on screen, with model, equation, variable units, and full-precision numeric cells. It omits ANOVA and advanced goodness-of-fit analysis. The adjacent `...` menu offers **Copy Full Fit Report**, also available in the parameter context menu. Both formats preserve manual/fixed parameter semantics and uncertainty units. Specified measurement uncertainty and identifiability warnings remain visible in the compact report when applicable. + +The uncertainty controls label the default **Unweighted - estimate from residuals**. Known uncertainty entry uses a wider field on a separate row. From the default mode, pixels start at 1; physical units start at one calibrated pixel, or the residual standard error if calibration is unavailable. A zero/undefined starting scatter leaves the entry blank. Switching between supplied physical and pixel modes converts the current value. Pixel presets are offered only in pixel mode. These starting values remain editable measurement-model choices, not claims about instrument accuracy. diff --git a/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java b/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java index c24d6f9b..e957707f 100644 --- a/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java +++ b/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java @@ -52,6 +52,13 @@ public static void main(String[] args)throws Exception { java.lang.reflect.Field uf=DatasetCurveFitter.class.getDeclaredField("uncertaintyPanel");uf.setAccessible(true);JPanel uncertainty=(JPanel)uf.get(fitter); Rectangle uncertaintyBounds=SwingUtilities.convertRectangle(uncertainty,new Rectangle(0,0,uncertainty.getWidth(),uncertainty.getHeight()),fitter); check(fitter.getVisibleRect().contains(uncertaintyBounds),"uncertainty controls visible in fit area"); + for (String fieldName : new String[]{"uncertaintyChoice", "uncertaintyValue"}) { + java.lang.reflect.Field field=DatasetCurveFitter.class.getDeclaredField(fieldName);field.setAccessible(true); + JComboBox control=(JComboBox)field.get(fitter); + Rectangle bounds=SwingUtilities.convertRectangle(control,new Rectangle(0,0,control.getWidth(),control.getHeight()),fitter); + check(fitter.getVisibleRect().contains(bounds),"uncertainty input fully visible: "+fieldName); + if(fieldName.equals("uncertaintyValue"))check(control.getWidth()>=control.getFontMetrics(control.getFont()).stringWidth("0.0000000000"),"room to enter fractional physical units"); + } check(tab.splitPanes[1].getTopComponent().getHeight()>100,"plot retains usable height"); check(Math.abs(fitter.getHeight()-fitter.prepareFitLayout(fitter.getWidth()))<=4,"fit area stays at required height"); if(args.length>0){BufferedImage image=new BufferedImage(tool.getContentPane().getWidth(),tool.getContentPane().getHeight(),BufferedImage.TYPE_INT_RGB);Graphics2D g=image.createGraphics();tool.getContentPane().printAll(g);g.dispose();javax.imageio.ImageIO.write(image,"png",new java.io.File(args[0]+"/data-tool-"+size[0]+"-font"+level+".png"));} diff --git a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java index 00d5345b..cc808e6e 100644 --- a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java +++ b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java @@ -117,6 +117,17 @@ static void run() { tab.setWorkingColumns("t","x"); fitter=tab.getCurveFitter();fitter.selectFit(fitter.getPolyFitNameOfDegree(1));fitter.setActiveAndFit(true); double measuredSlope=fitter.fit.getParameterValue(0); + fitter.selectUncertaintyMode(FitUncertainty.CONSTANT); + near(fitter.getUncertaintyModel().value,.01,"physical default is one calibrated pixel"); + fitter.setUncertaintyModel(FitUncertainty.CONSTANT,.001); + fitter.selectUncertaintyMode(FitUncertainty.PIXELS); + near(fitter.getUncertaintyModel().value,.1,"physical to pixel switch preserves scale"); + fitter.selectUncertaintyMode(FitUncertainty.CONSTANT); + near(fitter.getUncertaintyModel().value,.001,"pixel to physical switch preserves scale"); + fitter.selectUncertaintyMode(FitUncertainty.ESTIMATED); + fitter.selectUncertaintyMode(FitUncertainty.PIXELS); + near(fitter.getUncertaintyModel().value,1,"pixel default is one pixel"); + check(fitter.fit.getParameterValue(0)==measuredSlope,"mode switches preserve coefficient"); near(fitter.getYUnitsPerPixel(),.01,"host position calibration reaches fitter: "+fitter.getData().getYColumnName()); for(double pixels:new double[]{.5,1,1.5,2,2.5,3,.73}) { fitter.setUncertaintyModel(FitUncertainty.PIXELS,pixels); From 9c3d30b9e9222237660c3c66d60865222f776318 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Sun, 6 Sep 2026 23:07:22 -0400 Subject: [PATCH 10/21] Display data uncertainty with two significant digits --- .../tools/DatasetCurveFitter.java | 26 +++++++++++++++++-- test/fit-report-physics.md | 2 ++ .../tools/CurveFitPhysicsTest.java | 7 +++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index 1052f3d4..a041f0a3 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -246,6 +246,22 @@ public void refreshUncertaintyModel() { refreshFitStatistics(); refreshUncertaintyControls(); } + /** Two significant digits for display only; the model retains the original value. */ + static String formatDataUncertainty(double value) { + if (!FitUncertainty.positive(value)) return ""; + DecimalFormat format = new DecimalFormat("0.0E0", OSPRuntime.getDecimalFormatSymbols()); + String scientific = format.format(value); + int exponent = Integer.parseInt(scientific.substring(scientific.indexOf('E') + 1)); + if (exponent < -3 || exponent > 1) return scientific; + String pattern = "0"; + if (exponent < 1) { + pattern += "."; + for (int i = 0; i < 1 - exponent; i++) pattern += "0"; + } + format.applyPattern(pattern); + return format.format(value); + } + private void refreshUncertaintyControls() { if(uncertaintyChoice==null)return; updatingUncertaintyControls=true; @@ -260,8 +276,11 @@ private void refreshUncertaintyControls() { if (uncertaintyModel.mode == FitUncertainty.PIXELS) for (String preset : new String[]{"0.5", "1.0", "1.5", "2.0", "2.5", "3.0"}) uncertaintyValue.addItem(preset); if(uncertaintyModel.mode!=FitUncertainty.ESTIMATED)uncertaintyValue.setSelectedItem(FitUncertainty.positive(uncertaintyModel.value) - ? Double.toString(uncertaintyModel.value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : ""); + ? formatDataUncertainty(uncertaintyModel.value) : ""); else uncertaintyValue.setSelectedItem(""); + uncertaintyValue.setToolTipText(ToolsRes.getString("DatasetCurveFitter.Uncertainty.Custom") + + (uncertaintyModel.mode != FitUncertainty.ESTIMATED && FitUncertainty.positive(uncertaintyModel.value) + ? " " + Double.toString(uncertaintyModel.value).replace('.', OSPRuntime.getCurrentDecimalSeparator()) : "")); double sigma=uncertaintyModel.sigma(Double.NaN,getYUnitsPerPixel()); uncertaintyStatus.setText(uncertaintyModel.mode==FitUncertainty.ESTIMATED?"": FitUncertainty.positive(sigma)?(uncertaintyModel.mode == FitUncertainty.PIXELS ? "pixels" @@ -1202,8 +1221,11 @@ public void layoutContainer(Container target) { uncertaintyStatus=new JLabel(); java.awt.event.ActionListener changed=e->{ if(updatingUncertaintyControls)return; + String entered = uncertaintyValue.getEditor().getItem().toString().trim(); + // Enter/focus events on unchanged display text must not round the stored scale. + if (entered.equals(formatDataUncertainty(uncertaintyModel.value))) return; double value=Double.NaN; - try { value=Double.parseDouble(uncertaintyValue.getEditor().getItem().toString().trim().replace(OSPRuntime.getCurrentDecimalSeparator(),'.')); } + try { value=Double.parseDouble(entered.replace(OSPRuntime.getCurrentDecimalSeparator(),'.')); } catch(Exception ex) { /* Keep invalid supplied input unavailable, never estimate it. */ } setUncertaintyModel(uncertaintyChoice.getSelectedIndex(),value); }; diff --git a/test/fit-report-physics.md b/test/fit-report-physics.md index c5417c8b..1a9ac639 100644 --- a/test/fit-report-physics.md +++ b/test/fit-report-physics.md @@ -92,3 +92,5 @@ native Windows/Linux runtime testing. Copy Fit Report exports the parameter table and the statistics shown on screen, with model, equation, variable units, and full-precision numeric cells. It omits ANOVA and advanced goodness-of-fit analysis. The adjacent `...` menu offers **Copy Full Fit Report**, also available in the parameter context menu. Both formats preserve manual/fixed parameter semantics and uncertainty units. Specified measurement uncertainty and identifiability warnings remain visible in the compact report when applicable. The uncertainty controls label the default **Unweighted - estimate from residuals**. Known uncertainty entry uses a wider field on a separate row. From the default mode, pixels start at 1; physical units start at one calibrated pixel, or the residual standard error if calibration is unavailable. A zero/undefined starting scatter leaves the entry blank. Switching between supplied physical and pixel modes converts the current value. Pixel presets are offered only in pixel mode. These starting values remain editable measurement-model choices, not claims about instrument accuracy. + +The data-uncertainty entry displays two significant digits (for example, `0.0025 m`). Its full stored precision remains in the tooltip and exported reports. Formatting and accepting unchanged display text do not round the uncertainty used in calculations. diff --git a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java index cc808e6e..218c648c 100644 --- a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java +++ b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java @@ -16,6 +16,13 @@ public static void main(String[] args) throws Exception { if(!OSPRuntime.isJS)System.exit(0); } static void run() { + OSPRuntime.setPreferredDecimalSeparator("."); + check(DatasetCurveFitter.formatDataUncertainty(.002518020744556).equals("0.0025"),"data uncertainty two significant digits"); + check(DatasetCurveFitter.formatDataUncertainty(1).equals("1.0"),"pixel trailing zero"); + check(DatasetCurveFitter.formatDataUncertainty(.000001234).equals("1.2E-6"),"small uncertainty compact scientific notation"); + check(DatasetCurveFitter.formatDataUncertainty(.00999).equals("0.010"),"uncertainty rounding carry"); + OSPRuntime.setPreferredDecimalSeparator(","); + check(DatasetCurveFitter.formatDataUncertainty(.002518).equals("0,0025"),"localized uncertainty display"); OSPRuntime.setPreferredDecimalSeparator("."); DatasetManager manager=new DatasetManager();manager.setXPointsLinked(true); manager.setXYColumnNames(0,"t","x");manager.append(0,new double[]{0,1,2,3},new double[]{1.1,2.9,4.9,7.1}); From f76b42aed3de4fb237c9c62196dab1826a7db534 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 10:55:31 -0400 Subject: [PATCH 11/21] Compact fit controls and export motion results from position fits --- .../resources/tools/tools.properties | 6 +++ .../tools/CurveFitReport.java | 32 ++++++++++++++++ .../opensourcephysics/tools/DataToolTab.java | 38 +++++++++++++------ .../tools/DatasetCurveFitter.java | 32 ++++++++-------- .../tools/FitMetadataProvider.java | 4 ++ test/fit-report-physics.md | 10 +++++ .../tools/CurveFitDataToolLayoutTest.java | 24 +++++++++--- .../tools/CurveFitPhysicsTest.java | 25 ++++++++++++ .../tools/CurveFitPopupTest.java | 4 +- 9 files changed, 140 insertions(+), 35 deletions(-) diff --git a/src/org/opensourcephysics/resources/tools/tools.properties b/src/org/opensourcephysics/resources/tools/tools.properties index cfe91f06..fa74af79 100644 --- a/src/org/opensourcephysics/resources/tools/tools.properties +++ b/src/org/opensourcephysics/resources/tools/tools.properties @@ -992,3 +992,9 @@ DatasetCurveFitter.Menuitem.CopyFullReport=Copy Full Fit Report DatasetCurveFitter.Button.CopyOptions.Tooltip=More copy options (full fit analysis) DatasetCurveFitter.Uncertainty.Value=Uncertainty value: + +DatasetCurveFitter.Report.Velocity=velocity +DatasetCurveFitter.Report.VelocityAtZero=velocity at t=0 +DatasetCurveFitter.Report.Acceleration=acceleration +DatasetCurveFitter.Report.MotionResults=MOTION FROM POSITION FIT +DatasetCurveFitter.Report.Quantity=Quantity diff --git a/src/org/opensourcephysics/tools/CurveFitReport.java b/src/org/opensourcephysics/tools/CurveFitReport.java index 1e67b0eb..e7ff08f3 100644 --- a/src/org/opensourcephysics/tools/CurveFitReport.java +++ b/src/org/opensourcephysics/tools/CurveFitReport.java @@ -113,6 +113,38 @@ static String createSummary(KnownFunction fit, Dataset data, boolean[] fixed, return out.toString(); } + /** Exact transformations of polynomial coefficients; no new fit or covariance approximation. */ + static String[][] motionResults(KnownFunction fit, boolean[] fixed, double[] uncertainties, + boolean autofit, String xUnits, String yUnits, String component) { + if (component == null || !(fit instanceof KnownPolynomial) + || fit.getParameterCount() < 2 || fit.getParameterCount() > 3) return new String[0][]; + int degree = fit.getParameterCount() - 1; + String[][] results = new String[degree][]; + for (int i = 0; i < degree; i++) { + int parameter = degree == 1 ? 0 : 1 - i; + double factor = i == 0 ? 1 : 2; + boolean isFixed = fixed != null && parameter < fixed.length && fixed[parameter]; + double sigma = autofit && !isFixed && uncertainties != null && parameter < uncertainties.length + ? factor * uncertainties[parameter] : Double.NaN; + if (sigma < 0) sigma = Double.NaN; + results[i] = new String[]{component + " " + label(i == 1 ? "Acceleration" + : degree == 1 ? "Velocity" : "VelocityAtZero"), + number(factor * fit.getParameterValue(parameter)), number(sigma), + coefficientUnits(fit, parameter, xUnits, yUnits), label(isFixed ? "Yes" : "No")}; + } + return results; + } + + static String appendMotionResults(String report, String[][] results) { + if (results.length == 0) return report; + StringBuilder out = new StringBuilder(report); + row(out); + row(out, label("MotionResults")); + row(out, label("Quantity"), label("Value"), label("StandardError"), label("Units"), label("Fixed")); + for (String[] result : results) row(out, result); + return out.toString(); + } + static String coefficientUnits(KnownFunction fit,int parameter,String xUnits,String yUnits) { if(!(fit instanceof KnownPolynomial))return ""; String x=org.opensourcephysics.display.ExportText.ascii(xUnits).trim(); diff --git a/src/org/opensourcephysics/tools/DataToolTab.java b/src/org/opensourcephysics/tools/DataToolTab.java index 9889b4e4..3ba9f0f5 100644 --- a/src/org/opensourcephysics/tools/DataToolTab.java +++ b/src/org/opensourcephysics/tools/DataToolTab.java @@ -196,6 +196,7 @@ public void setFitMetadataProvider(FitMetadataProvider provider) { protected DataToolStatsTable statsTable; protected DataToolPropsTable propsTable; protected JScrollPane dataScroller, statsScroller, propsScroller, tableScroller; + protected JScrollPane fitScroller; protected JToolBar toolbar; protected JCheckBoxMenuItem statsCheckbox, propsCheckbox, fourierCheckbox; protected FourierPanel fourierPanel; @@ -1199,14 +1200,22 @@ public void propertyChange(PropertyChangeEvent e) { splitPanes[1] = new JSplitPane(JSplitPane.VERTICAL_SPLIT) { @Override public void doLayout() { - if (getBottomComponent() instanceof DatasetCurveFitter && getHeight() > 0) { - DatasetCurveFitter fitter = (DatasetCurveFitter) getBottomComponent(); - java.awt.Insets insets = getInsets(); - int width = getWidth() - insets.left - insets.right; - int required = fitter.prepareFitLayout(width); - setDividerLocation(Math.max(insets.top, - getHeight() - insets.bottom - getDividerSize() - required)); - } + if (fitScroller != null && getBottomComponent() == fitScroller && getHeight() > 0) { + java.awt.Insets insets = getInsets(); + // Reserve scrollbar width consistently so entering scroll mode does + // not toggle the responsive layout back and forth. + int width = Math.max(1, getWidth() - insets.left - insets.right + - fitScroller.getVerticalScrollBar().getPreferredSize().width); + int required = curveFitter.prepareFitLayout(width); + curveFitter.setPreferredSize(new Dimension(width, required)); + int available = getHeight() - insets.top - insets.bottom - getDividerSize(); + // Protect the drawable graph, not merely its panel and axis labels. + int graphHeight = Math.max(140, plot.getFontMetrics(plot.getFont()).getHeight() * 8); + int plotHeight = graphHeight + plot.getTopGutter() + plot.getBottomGutter(); + plotHeight = Math.min(plotHeight, Math.max(0, available / 2)); + int fitHeight = Math.min(required, Math.max(0, available - plotHeight)); + setDividerLocation(insets.top + available - fitHeight); + } super.doLayout(); } @@ -1311,7 +1320,14 @@ public void columnMoved(TableColumnModelEvent e) { public void actionPerformed(ActionEvent e) { splitPanes[1].setEnabled(true); getCurveFitter().setFontLevel(FontSizer.getLevel()); - splitPanes[1].setBottomComponent(curveFitter); + if (fitScroller == null) { + fitScroller = new JScrollPane(curveFitter, + JScrollPane.VERTICAL_SCROLLBAR_ALWAYS, JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED); + fitScroller.setBorder(BorderFactory.createEmptyBorder()); + fitScroller.setMinimumSize(new Dimension(0, 0)); + fitScroller.getVerticalScrollBar().setUnitIncrement(16); + } + splitPanes[1].setBottomComponent(fitScroller); splitPanes[1].setDividerSize(splitPanes[0].getDividerSize()); splitPanes[1].setDividerLocation(-1); if (curveFitter.getDrawer() != null) @@ -1334,7 +1350,7 @@ public void actionPerformed(ActionEvent e) { @Override public void actionPerformed(ActionEvent e) { // remove curveFitter - splitPanes[1].remove(curveFitter); + if (fitScroller != null) splitPanes[1].remove(fitScroller); splitPanes[1].setDividerSize(splitPanes[2].getDividerSize()); splitPanes[1].setDividerLocation(1.0); plot.removeDrawables(FunctionDrawer.class); @@ -2397,7 +2413,7 @@ public void actionPerformed(ActionEvent e) { private boolean isFitterVisible() { // return splitPanes[1].getDividerLocation() <= splitPanes[1].getMaximumDividerLocation(); - return curveFitter != null && splitPanes[1].getBottomComponent() == curveFitter; + return curveFitter != null && fitScroller != null && splitPanes[1].getBottomComponent() == fitScroller; } private void findHits(boolean showInTable) { diff --git a/src/org/opensourcephysics/tools/DatasetCurveFitter.java b/src/org/opensourcephysics/tools/DatasetCurveFitter.java index a041f0a3..07ab2e74 100644 --- a/src/org/opensourcephysics/tools/DatasetCurveFitter.java +++ b/src/org/opensourcephysics/tools/DatasetCurveFitter.java @@ -359,7 +359,7 @@ public void setAutoFit(boolean autofit) { private NumberField rmsField; private JPanel statisticsPanel, uncertaintyPanel; private JLabel[] statisticLabels; - private static final int[] VISIBLE_STATISTICS = {0, 1, 2, 5, 3, 6}; + private static final int[] VISIBLE_STATISTICS = {5, 3, 6}; private ParamTable paramTable; private ParamCellRenderer cellRenderer; private SpinCellEditor spinCellEditor; // uses number-crawler spinner @@ -687,6 +687,16 @@ private void refreshFitStatistics() { } } + private String[][] motionResults() { + FitMetadataProvider provider = tab == null ? null : tab.getFitMetadataProvider(); + String component = provider == null || dataset == null ? null + : provider.getPositionComponent(dataset.getXColumnName(), dataset.getYColumnName()); + double[] errors = fit == null ? new double[0] : new double[fit.getParameterCount()]; + for (int i = 0; i < errors.length; i++) errors[i] = getUncertainty(i); + return CurveFitReport.motionResults(fit, fixedParams.get(fit), errors, autofit, + variableUnits(true), variableUnits(false), component); + } + /** Copies a snapshot of the current fit without fitting or rounding its data. */ private void copyFitResults(boolean includeStatistics) { copyFitResults(includeStatistics, false); @@ -703,11 +713,12 @@ private void copyFitResults(boolean includeStatistics, boolean fullReport) { double[] sigma = new double[fit.getParameterCount()]; for (int i = 0; i < sigma.length; i++) sigma[i] = getUncertainty(i); - OSPRuntime.copy(includeStatistics && !fullReport + String report = includeStatistics && !fullReport ? CurveFitReport.createSummary(fit, dataset, fixedParams.get(fit), sigma, autofit, variableUnits(true), variableUnits(false), uncertaintyModel, getYUnitsPerPixel()) : CurveFitReport.create(fit, dataset, fixedParams.get(fit), sigma, - autofit, includeStatistics, variableUnits(true), variableUnits(false), uncertaintyModel, getYUnitsPerPixel()), null); + autofit, includeStatistics, variableUnits(true), variableUnits(false), uncertaintyModel, getYUnitsPerPixel()); + OSPRuntime.copy(includeStatistics ? CurveFitReport.appendMotionResults(report, motionResults()) : report, null); } /** @@ -1198,7 +1209,7 @@ public void layoutContainer(Container target) { copyButtons.add(copyFitOptionsButton, BorderLayout.EAST); rmsBar.add(copyButtons); rmsPanel.add(rmsBar, BorderLayout.NORTH); - statisticsPanel = new JPanel(new java.awt.GridLayout(0, 2, 10, 3)); + statisticsPanel = new JPanel(new java.awt.GridLayout(1, 3, 10, 0)); statisticsPanel.setBorder(BorderFactory.createEmptyBorder(4, 4, 4, 4)); statisticLabels = new JLabel[VISIBLE_STATISTICS.length]; for (int i = 0; i < statisticLabels.length; i++) { @@ -1488,19 +1499,6 @@ int prepareFitLayout(int availableWidth) { splitPane.setOrientation(orientation); splitPane.setResizeWeight(stacked ? 0 : 1); } - int labelWidth = 0; - for (JLabel label : statisticLabels) { - String text = label.getText(); - int colon = text.indexOf(':'); - String reserved = (colon < 0 ? text : text.substring(0, colon + 1)) + " -0.00000E-000"; - labelWidth = Math.max(labelWidth, label.getFontMetrics(label.getFont()).stringWidth(reserved)); - } - java.awt.GridLayout layout = (java.awt.GridLayout) statisticsPanel.getLayout(); - int columns = width < 2 * labelWidth + 18 ? 1 : 2; - if (layout.getColumns() != columns) { - layout.setColumns(columns); - statisticsPanel.revalidate(); - } int controlsHeight = splitPane.getLeftComponent().getPreferredSize().height; int parameterHeight = splitPane.getRightComponent().getPreferredSize().height; int infoHeight = (stacked ? controlsHeight + parameterHeight + splitPane.getDividerSize() diff --git a/src/org/opensourcephysics/tools/FitMetadataProvider.java b/src/org/opensourcephysics/tools/FitMetadataProvider.java index a5359f23..770ae354 100644 --- a/src/org/opensourcephysics/tools/FitMetadataProvider.java +++ b/src/org/opensourcephysics/tools/FitMetadataProvider.java @@ -6,6 +6,10 @@ public interface FitMetadataProvider { /** Null means no host metadata: retain DataTable's existing units. */ String getUnits(String columnName); + /** Position component (for example x or y) when the columns are position vs time. + * Null disables physical derivative labels. Hosts identify columns from metadata. + */ + default String getPositionComponent(String independentColumn, String dependentColumn) { return null; } /** Physical y units per image pixel, only for a directly calibrated position. * Return NaN for derived data, missing calibration, or varying conversion. */ diff --git a/test/fit-report-physics.md b/test/fit-report-physics.md index 1a9ac639..a9e84928 100644 --- a/test/fit-report-physics.md +++ b/test/fit-report-physics.md @@ -94,3 +94,13 @@ Copy Fit Report exports the parameter table and the statistics shown on screen, The uncertainty controls label the default **Unweighted - estimate from residuals**. Known uncertainty entry uses a wider field on a separate row. From the default mode, pixels start at 1; physical units start at one calibrated pixel, or the residual standard error if calibration is unavailable. A zero/undefined starting scatter leaves the entry blank. Switching between supplied physical and pixel modes converts the current value. Pixel presets are offered only in pixel mode. These starting values remain editable measurement-model choices, not claims about instrument accuracy. The data-uncertainty entry displays two significant digits (for example, `0.0025 m`). Its full stored precision remains in the tooltip and exported reports. Formatting and accepting unchanged display text do not round the uncertainty used in calculations. + +### Motion results from position fits + +An optional `FitMetadataProvider.getPositionComponent(xColumn, yColumn)` callback identifies position against time using host column identities. Its default is null, preserving existing hosts. For known line fits, velocity is A and its standard error is sigma_A. For known quadratics, velocity at t=0 is B with sigma_B, and constant acceleration is 2A with standard error 2*sigma_A. These are exact coefficient transformations; no new optimizer or covariance method is used. The units come from the corresponding polynomial coefficient units. Manual and fixed-parameter errors remain N/A. Arbitrary UserFunctions, other polynomial degrees, and unidentified column pairs do not receive motion labels. + +Derived motion results appear at the end of both copied report formats with full-precision numeric export. The window shows the original parameter table without duplicate velocity and acceleration rows. The reference time is the data's t=0, not necessarily the first selected observation. This feature does not compute velocity uncertainty at other times or propagate pixel errors into numerical derivative columns. + +At small window sizes the fit panel scrolls vertically, preserving a usable drawable graph above it. The reserved scrollbar width avoids changing the parameter layout when scrolling becomes necessary. Layout tests now measure the drawable area after axis gutters and verify that scrolling reaches the uncertainty controls. + +The on-screen residual summary uses one row: R-squared, SSE, and Residual SE. Points, free parameters, and degrees of freedom remain in copied reports but no longer consume display space. RMS remains beside Autofit. diff --git a/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java b/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java index e957707f..fe322c99 100644 --- a/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java +++ b/test/org/opensourcephysics/tools/CurveFitDataToolLayoutTest.java @@ -15,6 +15,11 @@ public static void main(String[] args)throws Exception { for(int i=0;i<16;i++){double t=i/30.0;data.append(t,-1980*t*t+1067*t-3.9+0.4*Math.sin(i));} DataTool tool=new DataTool(data); tool.addNotify(); DataToolTab tab=tool.getTab(0); tab.checkGUI(); + tab.setFitMetadataProvider(new FitMetadataProvider() { + public String getUnits(String column){return column.equals("t")?"s":"m";} + public double getYUnitsPerPixel(String column){return Double.NaN;} + public String getPositionComponent(String x,String y){return x.equals("t") && y.equals("y")?"y":null;} + }); DatasetCurveFitter fitter=tab.getCurveFitter(); tab.showFitterAction.actionPerformed(new ActionEvent(tab,0,fitter.getPolyFitNameOfDegree(2))); java.lang.reflect.Field sf=DatasetCurveFitter.class.getDeclaredField("statisticsPanel");sf.setAccessible(true);JPanel stats=(JPanel)sf.get(fitter); @@ -42,30 +47,39 @@ public static void main(String[] args)throws Exception { Rectangle text=new Rectangle(cell.x,cell.y,w,cell.height); check(table.getVisibleRect().contains(text),"parameter visible in nested viewport, font="+level+", width="+size[0]+", row="+r); Rectangle inFitter=SwingUtilities.convertRectangle(table,text,fitter); - check(fitter.getVisibleRect().contains(inFitter),"parameter visible in parent fit area"); + check(new Rectangle(0,0,fitter.getWidth(),fitter.getHeight()).contains(inFitter),"parameter visible in parent fit area"); } Rectangle parameterBottom=SwingUtilities.convertRectangle(table,table.getCellRect(2,2,true),fitter); Rectangle statsBounds=SwingUtilities.convertRectangle(stats,new Rectangle(0,0,stats.getWidth(),stats.getHeight()),fitter); check(statsBounds.y>=parameterBottom.y+parameterBottom.height,"parameter information above statistics"); - check(fitter.getVisibleRect().contains(statsBounds),"all statistics visible in parent fit area"); + check(new Rectangle(0,0,fitter.getWidth(),fitter.getHeight()).contains(statsBounds),"all statistics visible in parent fit area"); check(stats.getHeight()==stats.getPreferredSize().height,"statistics remain compact"); + check(stats.getComponentCount()==3,"only three statistics displayed"); + int statY=stats.getComponent(0).getY(); + for(Component label:stats.getComponents())check(label.getY()==statY,"statistics share a single row"); java.lang.reflect.Field uf=DatasetCurveFitter.class.getDeclaredField("uncertaintyPanel");uf.setAccessible(true);JPanel uncertainty=(JPanel)uf.get(fitter); Rectangle uncertaintyBounds=SwingUtilities.convertRectangle(uncertainty,new Rectangle(0,0,uncertainty.getWidth(),uncertainty.getHeight()),fitter); - check(fitter.getVisibleRect().contains(uncertaintyBounds),"uncertainty controls visible in fit area"); + check(new Rectangle(0,0,fitter.getWidth(),fitter.getHeight()).contains(uncertaintyBounds),"uncertainty controls visible in fit area"); for (String fieldName : new String[]{"uncertaintyChoice", "uncertaintyValue"}) { java.lang.reflect.Field field=DatasetCurveFitter.class.getDeclaredField(fieldName);field.setAccessible(true); JComboBox control=(JComboBox)field.get(fitter); Rectangle bounds=SwingUtilities.convertRectangle(control,new Rectangle(0,0,control.getWidth(),control.getHeight()),fitter); - check(fitter.getVisibleRect().contains(bounds),"uncertainty input fully visible: "+fieldName); + check(new Rectangle(0,0,fitter.getWidth(),fitter.getHeight()).contains(bounds),"uncertainty input fully visible: "+fieldName); if(fieldName.equals("uncertaintyValue"))check(control.getWidth()>=control.getFontMetrics(control.getFont()).stringWidth("0.0000000000"),"room to enter fractional physical units"); } check(tab.splitPanes[1].getTopComponent().getHeight()>100,"plot retains usable height"); + check(tab.plot.getHeight()-tab.plot.getTopGutter()-tab.plot.getBottomGutter()>=140,"drawable graph retains at least 140 pixels"); + check(tab.fitScroller.getViewport().getView()==fitter,"fit controls reachable through scroll viewport"); + tab.fitScroller.getVerticalScrollBar().setValue(tab.fitScroller.getVerticalScrollBar().getMaximum()); + layout(tool.getContentPane()); + check(fitter.getVisibleRect().contains(uncertaintyBounds),"scrolling reaches uncertainty controls"); + tab.fitScroller.getVerticalScrollBar().setValue(0); check(Math.abs(fitter.getHeight()-fitter.prepareFitLayout(fitter.getWidth()))<=4,"fit area stays at required height"); if(args.length>0){BufferedImage image=new BufferedImage(tool.getContentPane().getWidth(),tool.getContentPane().getHeight(),BufferedImage.TYPE_INT_RGB);Graphics2D g=image.createGraphics();tool.getContentPane().printAll(g);g.dispose();javax.imageio.ImageIO.write(image,"png",new java.io.File(args[0]+"/data-tool-"+size[0]+"-font"+level+".png"));} } } System.out.println("Passed: "+passed); success=true; - }catch(Exception e){throw new RuntimeException(e);}finally{System.exit(success ? 0 : 1);}}); + }catch(Throwable e){e.printStackTrace();}finally{System.exit(success ? 0 : 1);}}); System.exit(0); } static void layout(Container c){c.doLayout();for(Component child:c.getComponents())if(child instanceof Container)layout((Container)child);} diff --git a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java index 218c648c..78176c73 100644 --- a/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java +++ b/test/org/opensourcephysics/tools/CurveFitPhysicsTest.java @@ -24,6 +24,31 @@ static void run() { OSPRuntime.setPreferredDecimalSeparator(","); check(DatasetCurveFitter.formatDataUncertainty(.002518).equals("0,0025"),"localized uncertainty display"); OSPRuntime.setPreferredDecimalSeparator("."); + KnownPolynomial motion = new KnownPolynomial(new double[]{-.0098,2.688,-4.989}); + double[] motionErrors = {.023,.013,.0018}; + String[][] derived = CurveFitReport.motionResults(motion,null,motionErrors,true,"s","m","y"); + check(derived.length==2,"quadratic provides velocity at zero and acceleration"); + check(derived[0][0].equals("y velocity at t=0"),"reference time explicit"); + near(Double.parseDouble(derived[0][1]),2.688,"velocity is B"); + near(Double.parseDouble(derived[0][2]),.013,"velocity error is sigma B"); + near(Double.parseDouble(derived[1][1]),-9.978,"acceleration is twice A"); + near(Double.parseDouble(derived[1][2]),.046,"acceleration error is twice sigma A"); + check(derived[0][3].equals("m/s") && derived[1][3].equals("m/s^2"),"motion result units"); + String copied = CurveFitReport.appendMotionResults("",derived); + check(copied.contains("y acceleration\t-9.978\t0.046\tm/s^2"),"motion values exported once at full precision"); + check(CurveFitReport.motionResults(motion,null,motionErrors,true,"s","m",null).length==0,"no motion labels without host metadata"); + check(CurveFitReport.motionResults(new UserFunction("Arbitrary"),null,null,true,"s","m","y").length==0,"no guessed user-function motion"); + check(CurveFitReport.motionResults(new KnownPolynomial(new double[]{1,2,3,4}),null,null,true,"s","m","y").length==0,"no constant acceleration for cubic"); + derived=CurveFitReport.motionResults(motion,new boolean[]{true,false,false},motionErrors,true,"s","m","y"); + check(derived[1][2].equals("N/A") && derived[1][4].equals("Yes"),"fixed acceleration has no estimated error"); + derived=CurveFitReport.motionResults(motion,null,motionErrors,false,"s","m","y"); + check(derived[0][2].equals("N/A") && derived[1][2].equals("N/A"),"manual motion does not reuse fitted errors"); + derived=CurveFitReport.motionResults(new KnownPolynomial(new double[]{7,-2}),null,new double[]{.03,.2},true,"s","m","x"); + check(derived.length==1 && derived[0][0].equals("x velocity"),"line has velocity only"); + near(Double.parseDouble(derived[0][1]),-2,"line velocity sign and ordering"); + near(Double.parseDouble(derived[0][2]),.03,"line velocity uncertainty"); + near(motion.getParameterValue(0),-4.989,"derivation preserves coefficients"); + DatasetManager manager=new DatasetManager();manager.setXPointsLinked(true); manager.setXYColumnNames(0,"t","x");manager.append(0,new double[]{0,1,2,3},new double[]{1.1,2.9,4.9,7.1}); manager.setXYColumnNames(1,"t","v_{x}");manager.append(1,new double[]{0,1,2,3},new double[]{2,2,2,2}); diff --git a/test/org/opensourcephysics/tools/CurveFitPopupTest.java b/test/org/opensourcephysics/tools/CurveFitPopupTest.java index 25cfa3d6..dd75d5b0 100644 --- a/test/org/opensourcephysics/tools/CurveFitPopupTest.java +++ b/test/org/opensourcephysics/tools/CurveFitPopupTest.java @@ -20,8 +20,8 @@ public static void main(String[] args) throws Exception { fitter.setAutoFit(true); fitter.uncertainties = new double[] {.05, .01}; String labels = labels(fitter); - check(labels.contains("Points: 4"), "point count visible"); - check(labels.contains("Degrees of freedom: 2"), "degrees of freedom visible"); + check(!labels.contains("Points:"), "point count omitted from display"); + check(!labels.contains("Degrees of freedom:") && !labels.contains("Free parameters:"), "parameter counts omitted from display"); check(labels.contains("R-squared: 1.0000"), "R-squared visible"); check(labels.contains("SSE: 0.0000"), "SSE visible"); check(labels.contains("Residual SE: 0.0000"), "residual standard error visible"); From fea5ac3083de285366e106b5159a679edac17fcc Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 11:05:03 -0400 Subject: [PATCH 12/21] Explain fit uncertainties and document Bevington-style profiling --- test/README.md | 3 + test/bevington-fit-uncertainties.md | 279 ++++++++++++++++++++++++++++ test/fit-report-physics.md | 12 +- test/fit-uncertainty-guide.md | 131 +++++++++++++ 4 files changed, 420 insertions(+), 5 deletions(-) create mode 100644 test/bevington-fit-uncertainties.md create mode 100644 test/fit-uncertainty-guide.md diff --git a/test/README.md b/test/README.md index 1cee9f23..04b22b6d 100644 --- a/test/README.md +++ b/test/README.md @@ -1,5 +1,8 @@ # Curve-fit regression tests +For interpretation, see the [short uncertainty guide with examples](fit-uncertainty-guide.md). +For the numerical method, see the [Bevington-style implementation note](bevington-fit-uncertainties.md). + These standalone Java tests require a built OSP or Tracker JAR and a JDK. They use main methods and exit unsuccessfully on assertion failures; no test framework is required. Run from the repository root, replacing `/path/to/osp.jar` diff --git a/test/bevington-fit-uncertainties.md b/test/bevington-fit-uncertainties.md new file mode 100644 index 00000000..295f5aa8 --- /dev/null +++ b/test/bevington-fit-uncertainties.md @@ -0,0 +1,279 @@ +# Bevington-style fit uncertainties: implementation note + +This note documents the profile/refit curvature calculation in +[DatasetCurveFitter.java](../src/org/opensourcephysics/tools/DatasetCurveFitter.java) +as used in OSP PR #9. Start with the +[short guide and examples](fit-uncertainty-guide.md) for an introduction. + +The source identifies its uncertainty formula as Eq. 8.13 of *Data Reduction +and Error Analysis for the Physical Sciences*, associated with Philip R. +Bevington and, in later editions, D. Keith Robinson. The code does not specify +an edition or page. The equation-number attribution is inherited from that +comment; an edition-specific match has not been verified here. This note derives +the formula implemented in the code rather than reproducing the book or claiming +that every numerical choice below is prescribed by it. + +## 1. Objective and uncertainty scale + +For the selected valid observations, define + +```text +residual_i = y_i - f(x_i; parameters) +SSE = sum(residual_i^2) +chi^2 = SSE / sigma_y^2 +``` + +The current modes all use one common y uncertainty. Multiplying SSE by a +positive constant does not change its minimizing coefficients. The optimizer +therefore continues minimizing SSE, including when sigma_y is supplied. This +is mathematically constant-weight least squares; it does not imply support for +unequal per-observation weights. + +Let n be the number of selected valid observations, p the number of editable +free parameters, r their effective independent numerical rank, and df=n-r. +The uncertainty scale is: + +| Mode | Scale used | Interpretation of fit minimum | +|---|---|---| +| Estimated from residuals | sigma_y^2=SSE_min/df | chi^2_min=df; reduced chi^2=1 by construction, when defined | +| Supplied common sigma_y | user-supplied sigma_y^2 | chi^2_min=SSE_min/sigma_y^2; no residual rescaling | +| Supplied pixel sigma | (pixel sigma × physical units per pixel)^2 | same as supplied common sigma after conversion | + +For full-rank regression the first expression reduces to the usual residual +variance estimate with n-p in the denominator; see +[NIST's least-squares treatment](https://itl.nist.gov/div898/handbook/pmd/section4/pmd431.htm). +The rank substitution and the supplied-scale modes are implementation details +of this PR. Invalid supplied values are not replaced by a residual estimate. + +`calibrateChiSquared` establishes the scale once at the fitted minimum. +`getChiSquared` uses that same scale while parameters are perturbed. Re-estimating +sigma_y after every perturbation would erase the increase being measured. +The field `sigma_y_squared` holds a variance despite its older comment referring +to a standard deviation. + +## 2. Profile curvature and the factor of two + +For parameter a_j, define its profile objective conceptually as + +```text +P_j(a) = minimum chi^2 with a_j held at a, + refitting all other free parameters +``` + +Original user-fixed parameters remain fixed throughout. Near an identifiable +minimum a_hat, use a local quadratic approximation: + +```text +P_j(a_hat + h) ≈ chi^2_min + (1/2)*P_j''(a_hat)*h^2 + ≈ chi^2_min + h^2/sigma_j^2 +``` + +It follows that `sigma_j^2 ≈ 2/P_j''`. A symmetric finite difference gives + +```text +D = P_j(a_hat-delta) + P_j(a_hat+delta) - 2*chi^2_min +P_j'' ≈ D/delta^2 +sigma_j ≈ delta * sqrt(2/D) +``` + +D is the code's `twiceDeltaChiSq`: the sum of the two profile increases, not +one increase. Dropping the factor of two would underestimate the error by +sqrt(2) for a quadratic profile. + +Under this approximation, moving one parameter by one standard error raises +its profiled chi-square by about one. The implementation estimates a symmetric +local error; it does **not** search for exact left/right Delta-chi-square=1 +crossings or construct an asymmetric confidence interval. For a broader +account of profiling and likelihood-based errors, see the +[Particle Data Group statistics review](https://pdg.lbl.gov/2021/reviews/rpp2021-rev-statistics.pdf). +That review is background, not evidence for this implementation's step sizes. + +## 3. Why the other parameters must be refitted + +Consider `y=A*x+B`. Changing A generally requires changing B to keep the line +near the same observations. Holding B at its old fitted value measures a +conditional slice through the objective. Letting B adjust measures a profile. +These are different questions unless the parameters are uncoupled. + +For an exact quadratic objective, write its increment as `d^T H d`, where H is +half the Hessian of chi-square. Partition d into the tested displacement h and +other displacements z. Minimizing over z gives + +```text +z_best = -H_oo^(-1)*H_oj*h +Delta chi^2_profile = h^2*(H_jj - H_jo*H_oo^(-1)*H_oj) +sigma_j^2 = 1/(H_jj - H_jo*H_oo^(-1)*H_oj) = (H^(-1))_jj +``` + +Thus exact profiling and the inverse-curvature diagonal agree in a regular, +full-rank quadratic problem. For linear-in-parameter least squares this also +agrees with the usual covariance formula using the same uncertainty scale. +The PR preserves numerical profiling; it does not replace it with a Jacobian +covariance calculation. For nonlinear models, finite steps, nonquadratic +profiles, and imperfect minimization can produce differences. + +The rank calculation does evaluate a Jacobian. That is for counting independent +free directions, not for replacing the parameter-error method. + +## 4. A reproducible analytic example + +Take `x=[0,1,2,3]`, `y=[1.1,2.9,4.9,7.1]` and fit `y=A*x+B`. +The exact least-squares coefficients are A=2 and B=1, SSE=0.04, n=4, r=2, +df=2, and estimated sigma_y=sqrt(0.02). + +Here `x_mean=1.5` and `Sxx=sum((x-x_mean)^2)=5`. At a perturbed slope `2+h`, +the refitted intercept is `1-1.5*h`. Consequently, + +```text +SSE_profile(A) = 0.04 + 5*h^2 +chi^2_profile(A) = 2 + 250*h^2 +D = 500*delta^2 +sigma_A = sqrt(0.02/5) = 0.0632455532033676 +sigma_B = sqrt(0.02*(1/4 + 1.5^2/5)) = 0.1183215956619923 +``` + +Holding B fixed instead would use sum(x^2)=14 and give the smaller conditional +slope error `sqrt(0.02/14)=0.0377964473`. That is why the refit matters. + +If the independently supplied sigma_y is 0.20 instead, A and B are unchanged, +but `sigma_A=0.0894427191`, `sigma_B=0.1673320053`, chi-square=1, +reduced chi-square=0.5, and Q=exp(-1/2)=0.6065306597 for df=2. +With the residual-estimated scale, chi-square=2, reduced chi-square=1, +and Q is N/A. More generally, scaling a supplied sigma_y by c scales regular +profile errors by |c| and divides chi-square by c^2, up to numerical error. + +## 5. Code path and numerical procedure + +Relevant methods in `DatasetCurveFitter` are `fit`, `getTestFunction`, +`getUncertainties`, `calibrateChiSquared`, `getChiSquared`, +`refreshUncertaintyModel`, and `setUncertainties`. + +1. The existing coefficient fit runs. An unconstrained `KnownPolynomial` uses + its polynomial fitting routine. `UserFunction` fitting uses the existing + Hessian minimizer, with Levenberg-Marquardt attempted when that fit worsens + the objective; unsuccessful/worse results are restored. The current caller + uses 20 iterations and tolerance 1e-6 for these numerical fits. +2. `getTestFunction(f, fixedParams)` removes user-fixed parameters from the + free representation. `getUncertainties` requires at least one free parameter + and positive residual degrees of freedom, then establishes the scale. +3. Parameter names map the free representation back to the original order. + Missing/fixed parameter errors start as NaN. +4. For each free parameter with value a, start with + `delta=(abs(a)+1)/100000`. Construct two temporary `UserFunction` objects + representing that parameter held at a-delta and a+delta. The helper actually + substitutes the parameter name with a numeric literal in the expression; + it is not a general symbolic algebra engine. +5. Refit each temporary function's remaining parameters with `fit(test)` and + calculate D using the fixed uncertainty scale. Test functions are recognized + so they do not recursively launch their own uncertainty calculation. +6. Aim for `0.001 <= D <= 2`, with at most ten attempts. If the preceding D is + nonzero and below 0.001, multiply delta by ten; if above 2, divide delta by + ten. The first D is initialized to zero. Exactly zero D does not enlarge the + step in the current code, so a flat or cancellation-limited case can repeat + the same perturbation until the limit. Nonfinite D fails the final check. +7. After the loop, a finite D>=0.001 is accepted and the error is + `delta*sqrt(2/D)`. The upper target of 2 is not a final rejection condition + after the attempt limit. These thresholds are numerical heuristics, not + confidence levels or guaranteed convergence criteria. +8. For an accepted error, perform two more constrained refits at a±sigma_j. + Their parameter arrays are passed with the errors to the function drawer. + These are not a stored covariance matrix or a simultaneous confidence band. + +Each tested parameter can therefore request up to 20 shifted refits plus two +additional drawer refits. A model with one free parameter leaves no nuisance +parameter to optimize in its shifted functions. Runtime otherwise depends on +model evaluation, data size, and convergence of the numerical refits. + +Changing only the uncertainty choice calls `refreshUncertaintyModel`, which +clones the displayed function before recalculating its profile errors. The +original best-fit coefficients are preserved. The implementation reuses cached +temporary functions and the normal fit infrastructure; it is not an isolated, +side-effect-free statistical library. Tests cover the user-visible coefficient, +Autofit, fixed-parameter, and selection behavior. + +## 6. Rank, fixed parameters, and perfect fits + +[CurveFitReport.parameterRank](../src/org/opensourcephysics/tools/CurveFitReport.java) +uses normalized free-parameter derivative columns with pivoted, reorthogonalized +Gram-Schmidt and tolerance 1e-7. Polynomial derivatives are analytic; other +functions use central differences with step `1e-5*max(1,abs(parameter))`. +This is separate from the adaptive profile step above. For nonlinear models, +the rank is local and does not establish global identifiability. + +For `y=(A+B)*x`, the two derivative columns are identical. Four observations +have p=2, r=1, and df=3. Refitting B can compensate for a perturbation in A: +individual A and B errors should be unavailable. Rank drives df and the report's +warning; each coefficient error still depends on its own profile-curvature +check. The method does not replace a redundant fit with a uniquely identified +parameterization. + +A fixed coefficient reports N/A, not an estimated error of zero: its value is +an input constraint, and uncertainty in that input has not been propagated. +Manual mode likewise does not reuse automatic-fit inference. The current +implementation withholds parameter-error estimates when df<=0 even if a +supplied noise model could support inference in a different implementation. + +For exactly zero residual variance in estimated mode, the code temporarily +uses unscaled SSE to check profile curvature. An identifiable direction can +then receive zero estimated error; an unmeasurable direction remains NaN. +This is a consequence of the zero-scatter model, not proof of exact physical +knowledge. Other guards, including constant-response handling, can also make +errors unavailable. Exported chi-square remains N/A for the zero estimated +scale. A supplied positive sigma can give nonzero parameter errors even for +an exact fit; its chi-square is zero and Q=1 when df>0. + +## 7. Interpretation and limits + +The basic uncertainty model treats x as known and residual measurement errors +as independent with a common y variance. Gaussian errors and an adequate model +are needed for the usual chi-square probability interpretation. The report's +Q is the chi-square survival probability for the fitted residuals, not a +parameter p-value or the probability that the model is true. In estimated mode +Q is deliberately unavailable because the scale was learned from those residuals. + +For small samples, estimated standard errors are not automatically normal-based +confidence intervals; a regular linear model with unknown variance ordinarily +uses Student-t factors for coefficient intervals. For nonlinear, bounded, +weakly identified, or multimodal problems, a symmetric local curvature can be +misleading. Neither symmetric/asymmetric confidence intervals nor parameter +p-values are added by this PR. + +Numerical limitations include finite-difference cancellation, parameter scaling +(the additive 1 in the step rule is unit dependent), local minima, incomplete +refits, and strongly nonquadratic profiles. The accepted-D guard is not a +certificate of optimizer convergence. Very small/large uncertainty scales can +also underflow/overflow when squared. Existing tests exercise selected cases, +not all possible UserFunctions or parameterizations. + +Pixel conversion treats the calibration transform as given. It does not +propagate uncertain rulers, frame timing, lens distortion, common calibration +errors, or temporal correlations. The default residual estimate can absorb +marking noise and model inadequacy together. A large fitted error can make a +comparison with an expected value look reassuring while the measurement is +imprecise. Inspecting residuals and measurement choices remains necessary. + +Copied motion results use only exact single-coefficient transformations: +line-fit velocity A, quadratic velocity at t=0 B, and acceleration 2A with error +2*sigma_A. Velocity at another time, `2*A*t+B`, would require the A/B covariance +or a suitable reparameterized profile fit; it is not inferred by treating the +coefficient errors as independent. + +## 8. Verification and maintenance + +- [CurveFitPhysicsTest](org/opensourcephysics/tools/CurveFitPhysicsTest.java) + checks unchanged coefficients across uncertainty modes, supplied-scale error + scaling, UserFunction profile behavior, rank/df, perfect-fit cases, pixels, + and derived motion transformations. +- [CurveFitReportTest](org/opensourcephysics/tools/CurveFitReportTest.java) + includes the analytic four-observation line and report conventions. +- [CurveFitConstraintTest](org/opensourcephysics/tools/CurveFitConstraintTest.java) + checks fixed/manual behavior and redundant parameters. +- [CurveFitPrecisionTest](org/opensourcephysics/tools/CurveFitPrecisionTest.java) + separates full-precision values from display rounding. + +See [test instructions](README.md) and +[units, uncertainty modes, and integration](fit-report-physics.md). The same +computational fixtures run on the desktop and through SwingJS. The derivation +and worked numbers above are independently checkable algebra; they are not +claims that every theoretical profile is resolved exactly by the finite-step +implementation. This documentation makes no change to that implementation. diff --git a/test/fit-report-physics.md b/test/fit-report-physics.md index a9e84928..d908cca1 100644 --- a/test/fit-report-physics.md +++ b/test/fit-report-physics.md @@ -32,7 +32,9 @@ rescaled to force reduced chi-square to one. Parameter errors retain the Bevington Eq. 8.13 profile/refit curvature method. Only its uncertainty scale changes. This is not a replacement by Jacobian -covariance. Unidentifiable or fixed parameters retain unavailable errors. +covariance. Unidentifiable or fixed parameters retain unavailable errors. See the +[short explanation with examples](fit-uncertainty-guide.md) and the +[detailed implementation note](bevington-fit-uncertainties.md). Centered R Square = 1-SSE/SST, where SST uses deviations from the observed mean. Adjusted R Square = 1-(SSE/df)/(SST/(n-1)), where defined. R Square is descriptive @@ -78,11 +80,11 @@ a future column model must explicitly map uncertainties to the selected rows. ## Verification -The desktop suite contains 481 assertions across precision (19), report (94), -popup (17), Data Tool layout (174), constraints (40), and physics (137). -The companion Tracker calibration fixture adds 13 checks. The computational +The desktop suite contains 592 assertions across precision (19), report (118), +popup (17), Data Tool layout (234), constraints (40), and physics (164). +The companion Tracker calibration fixture adds 17 checks. The computational precision/report/constraint/physics tests also run through SwingJS in Chrome, -Firefox and WebKit (290 checks per browser). Browser interaction checks exercise +Firefox and WebKit (341 checks per browser). Browser interaction checks exercise selection, resizing, fixed parameters, supplied sigma, and copying the report. These local runs cover macOS and browser engines; they do not constitute new native Windows/Linux runtime testing. diff --git a/test/fit-uncertainty-guide.md b/test/fit-uncertainty-guide.md new file mode 100644 index 00000000..cbc8f7b8 --- /dev/null +++ b/test/fit-uncertainty-guide.md @@ -0,0 +1,131 @@ +# Understanding fit uncertainties + +A fit gives a best estimate and an uncertainty. They answer different questions: +**what value fits these measurements best, and how precisely do the measurements +constrain it?** A small uncertainty does not establish that the model or the +measurements are correct. + +This guide describes the Data Tool behavior in OSP PR #9. The +[technical note](bevington-fit-uncertainties.md) explains the implementation. + +## What the error beside a coefficient means + +Suppose a line describes position versus time: + +```text +x = A*t + B +``` + +The slope A is velocity; B is position at t=0. To investigate uncertainty in A, +Tracker moves the slope slightly away from its best value, allows B to adjust, +and checks how much worse the fit becomes. It repeats this on the other side. +If the fit becomes worse quickly, the data constrain A tightly. If many nearby +slopes fit almost as well, its uncertainty is larger. This is the idea behind +the profile/refit curvature method identified in the code with Bevington's +Eq. 8.13. + +The reported error is a **standard-error estimate**, not a maximum possible +error and not a 95% confidence interval. Its interpretation depends on the +measurement-error model and the adequacy of the fitted function. + +## A small example you can reproduce in a spreadsheet + +These are illustrative measurements, not a Tracker video: + +| Time (s) | Position (m) | +|---|---| +| 0 | 1.1 | +| 1 | 2.9 | +| 2 | 4.9 | +| 3 | 7.1 | + +The best line is `x = 2*t + 1`. The measured positions differ from it by +`+0.1, -0.1, -0.1, +0.1 m`, so the sum of squared residuals (SSE) is +`0.040 m^2`. + +With four observations and two independent fitted parameters, the residual +standard error is `sqrt(0.040/2) = 0.141421... m`. Using that estimated scatter, +the coefficient results are approximately: + +```text +Velocity A 2.000 +/- 0.063 m/s +Position B 1.00 +/- 0.12 m +``` + +The velocity uncertainty is not the same as the scatter of individual position +measurements. It depends on the measurement times as well as the position +scatter. A longer time span can constrain a slope more tightly, provided the +same model still applies. + +## Where the uncertainty scale comes from + +**Unweighted - estimate from residuals** is the default. All selected points +receive equal weight, and their scatter about the fit estimates the common +position uncertainty. This is a useful starting point when no independent +measurement uncertainty has been supplied. The usual residual-scatter estimate +is described in the [NIST least-squares guide](https://itl.nist.gov/div898/handbook/pmd/section4/pmd431.htm). + +**Known uncertainty (data units)** uses the positive uncertainty you supply. +For the same example, specifying `0.20 m` leaves the best line unchanged, but +makes the velocity uncertainty approximately `0.089 m/s`. The chosen uncertainty +scale has increased; the measured points have not changed. + +**Known uncertainty (pixels)** supplies the same kind of model through Tracker's +position calibration. If the calibration is `0.0025 m/pixel`, a choice of +`1.0 pixel` means `0.0025 m`; `0.5 pixel` means `0.00125 m`. One pixel is a +convenient starting choice, not an assertion about your marking accuracy. +The current conversion does not include uncertainty in the calibration itself. +Pixel mode is available only for eligible directly calibrated position data. + +## What agreement does—and does not—tell you + +Careless marking can increase the residual scatter and therefore increase +residual-estimated parameter errors. A result can then agree with an expected +value within its uncertainty because it is imprecise. A small discrepancy in +units of the reported error (often called a Z-value) does not, by itself, +establish good measurement technique. + +The full report offers two different checks: + +- **R-squared** describes how much of the observed variation the fitted model + accounts for. It does not establish that residuals are small relative to your + measurement uncertainty. +- **Reduced chi-square** compares residuals with the supplied uncertainty scale. + When Tracker estimates that scale from the same residuals, reduced chi-square + is 1 by construction whenever defined. It is then not an independent test, + and the chi-square probability Q is N/A. + +With a genuinely supplied common uncertainty, a reduced chi-square much above +one can indicate underestimated uncertainty, a poor model, or other violated +assumptions. A value well below one can indicate an overestimate or correlations. +These are reasons to inspect the experiment, not automatic diagnoses. + +## Position fits can give motion results directly + +For a line fit to position versus time, the slope and its error are velocity +and its error. For a quadratic `y = A*t^2 + B*t + C`, acceleration is `2*A` and +its standard error is `2*error(A)`. + +For example, if `A = -4.989 +/- 0.023 m/s^2`, the acceleration is +`-9.978 +/- 0.046 m/s^2`. The actual software uses unrounded values before +formatting the result. B gives velocity at the data's t=0, which need not be the +first selected frame. These derived quantities appear in copied reports when +Tracker identifies the columns as position versus time. + +The pixel setting does not propagate uncertainties into the separately +calculated velocity/acceleration columns. Those calculations reuse positions, +so their errors can be correlated. For this purpose, fitting the measured +position data avoids a second regression on differentiated data. + +## Reporting a result + +Keep the full precision for subsequent calculations; round at the reporting +stage. Tracker displays two significant digits in positive uncertainty values +and matches the coefficient's displayed decimal place to its uncertainty. +Copied numeric results retain full precision. + +Record the fitted interval, equation, units, and uncertainty choice with the +result. If an error is N/A, it is unavailable—not zero. Manual parameters, +fixed coefficients, insufficient information, and unidentifiable parameters +require particular care. The full report is available from **... → Copy Full +Fit Report** when you need the details. From b4e68ae022c70220d002c9c3444ee862d699f6f5 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 11:25:42 -0400 Subject: [PATCH 13/21] Clarify fitted values, measurement uncertainties, and coefficient errors --- test/bevington-fit-uncertainties.md | 140 ++++++++++++------ test/fit-uncertainty-guide.md | 219 +++++++++++++++------------- 2 files changed, 211 insertions(+), 148 deletions(-) diff --git a/test/bevington-fit-uncertainties.md b/test/bevington-fit-uncertainties.md index 295f5aa8..1e3340b6 100644 --- a/test/bevington-fit-uncertainties.md +++ b/test/bevington-fit-uncertainties.md @@ -1,9 +1,15 @@ # Bevington-style fit uncertainties: implementation note -This note documents the profile/refit curvature calculation in +Tracker first fits the coefficients of a function to the measured data. It then +estimates an error for each free coefficient by changing that coefficient, +refitting the others, and measuring the increase in chi-square. This second +calculation is the subject of this note. + +The implementation is in [DatasetCurveFitter.java](../src/org/opensourcephysics/tools/DatasetCurveFitter.java) -as used in OSP PR #9. Start with the -[short guide and examples](fit-uncertainty-guide.md) for an introduction. +as used in OSP PR #9. The [short guide](fit-uncertainty-guide.md) introduces the +ideas through a position-versus-time example. This note gives the equations, +code path, and numerical limitations. The source identifies its uncertainty formula as Eq. 8.13 of *Data Reduction and Error Analysis for the Physical Sciences*, associated with Philip R. @@ -13,8 +19,11 @@ comment; an edition-specific match has not been verified here. This note derives the formula implemented in the code rather than reproducing the book or claiming that every numerical choice below is prescribed by it. -## 1. Objective and uncertainty scale +## 1. The quantity minimized and the measurement uncertainty +An **objective function** is the quantity the fitting algorithm minimizes. +Here it is the sum of squared residuals, SSE. A residual is the difference +between a measured y value and the function's prediction at the same x. For the selected valid observations, define ```text @@ -23,15 +32,28 @@ SSE = sum(residual_i^2) chi^2 = SSE / sigma_y^2 ``` -The current modes all use one common y uncertainty. Multiplying SSE by a -positive constant does not change its minimizing coefficients. The optimizer -therefore continues minimizing SSE, including when sigma_y is supplied. This -is mathematically constant-weight least squares; it does not imply support for -unequal per-observation weights. +Two kinds of uncertainty occur below. `sigma_y` is the common standard +uncertainty assigned to the measured y values. `sigma_j` is the standard error +calculated for fitted coefficient j. The measurement uncertainty sigma_y is +an input to the coefficient-error calculation; sigma_j is one of its results. + +All current uncertainty modes assign the same sigma_y to every selected point. +Dividing SSE by this positive constant squared changes the size of the objective, +but not the coefficient values at its minimum. The optimizer therefore continues +minimizing SSE when sigma_y is supplied. This implements constant-weight least +squares; it does not provide unequal weights for different observations. + +The following notation separates the data count from the parameter count: -Let n be the number of selected valid observations, p the number of editable -free parameters, r their effective independent numerical rank, and df=n-r. -The uncertainty scale is: +| Symbol | Meaning | +|---|---| +| n | Number of selected valid observations | +| p | Number of coefficients the optimizer may change | +| r | Number of independent parameter directions, estimated numerically; see section 6 | +| df | Residual degrees of freedom, n-r | +| SSE_min | SSE evaluated at the fitted coefficients | + +The measurement uncertainty is obtained as follows: | Mode | Scale used | Interpretation of fit minimum | |---|---|---| @@ -45,30 +67,40 @@ variance estimate with n-p in the denominator; see The rank substitution and the supplied-scale modes are implementation details of this PR. Invalid supplied values are not replaced by a residual estimate. -`calibrateChiSquared` establishes the scale once at the fitted minimum. -`getChiSquared` uses that same scale while parameters are perturbed. Re-estimating -sigma_y after every perturbation would erase the increase being measured. +`calibrateChiSquared` calculates sigma_y once at the fitted coefficients. +`getChiSquared` keeps it fixed while testing nearby coefficient values. +If sigma_y were recalculated from the new residuals at each test value, the +normalization would hide the worsening fit that the calculation needs to measure. The field `sigma_y_squared` holds a variance despite its older comment referring to a standard deviation. ## 2. Profile curvature and the factor of two -For parameter a_j, define its profile objective conceptually as +Hold one coefficient at a trial value and refit the others. Repeat at other +trial values. The resulting minimum chi-square as a function of that one +coefficient is called its **profile**. Its **curvature** measures how quickly +chi-square rises as the tested coefficient moves away from its fitted value. + +For coefficient a_j, write the profile as ```text P_j(a) = minimum chi^2 with a_j held at a, refitting all other free parameters ``` -Original user-fixed parameters remain fixed throughout. Near an identifiable -minimum a_hat, use a local quadratic approximation: +Coefficients the user has fixed remain fixed throughout. Let a_hat be the +fitted value of the coefficient being tested, and h a small change in that +value. If the data determine that coefficient and the profile is approximately +quadratic near its minimum, then: ```text P_j(a_hat + h) ≈ chi^2_min + (1/2)*P_j''(a_hat)*h^2 ≈ chi^2_min + h^2/sigma_j^2 ``` -It follows that `sigma_j^2 ≈ 2/P_j''`. A symmetric finite difference gives +Here `P_j''` is the second derivative of the profile. Comparing the two +expressions gives `sigma_j^2 ≈ 2/P_j''`. The code estimates that second +derivative from two trial values, equally spaced by delta on either side: ```text D = P_j(a_hat-delta) + P_j(a_hat+delta) - 2*chi^2_min @@ -91,13 +123,22 @@ That review is background, not evidence for this implementation's step sizes. ## 3. Why the other parameters must be refitted Consider `y=A*x+B`. Changing A generally requires changing B to keep the line -near the same observations. Holding B at its old fitted value measures a -conditional slice through the objective. Letting B adjust measures a profile. -These are different questions unless the parameters are uncoupled. - -For an exact quadratic objective, write its increment as `d^T H d`, where H is -half the Hessian of chi-square. Partition d into the tested displacement h and -other displacements z. Minimizing over z gives +near the same observations. Holding B at its original fitted value asks how +precisely A is determined +*if B is already known*. Letting B adjust asks how precisely A is determined +*when B must also be estimated from these data*. The first calculation is called +a conditional slice; the second is a profile. They generally give different +errors because changes in A and B can partly compensate for each other. + +The matrix calculation below shows how profiling relates to covariance. +The worked example in section 4 illustrates the same distinction without matrices. + +For an exact quadratic objective, write the increase in chi-square as `d^T H d`. +Here d contains all coefficient changes, and H is half the Hessian—the matrix +of second derivatives of chi-square. Separate d into the tested coefficient's +change h and the other coefficients' changes z. Subscripts j and o below refer +to the tested coefficient and the other coefficients, respectively. Minimizing +over z gives ```text z_best = -H_oo^(-1)*H_oj*h @@ -112,8 +153,10 @@ The PR preserves numerical profiling; it does not replace it with a Jacobian covariance calculation. For nonlinear models, finite steps, nonquadratic profiles, and imperfect minimization can produce differences. -The rank calculation does evaluate a Jacobian. That is for counting independent -free directions, not for replacing the parameter-error method. +The separate rank calculation evaluates a Jacobian: a matrix describing how +the predicted data change when each coefficient changes. It uses that matrix +to count independent parameter directions. It does not use it to calculate +parameter errors. ## 4. A reproducible analytic example @@ -180,15 +223,15 @@ Relevant methods in `DatasetCurveFitter` are `fit`, `getTestFunction`, These are not a stored covariance matrix or a simultaneous confidence band. Each tested parameter can therefore request up to 20 shifted refits plus two -additional drawer refits. A model with one free parameter leaves no nuisance -parameter to optimize in its shifted functions. Runtime otherwise depends on +additional drawer refits. For a model with only one free coefficient, holding that coefficient at a +trial value leaves no other coefficient to refit. Runtime otherwise depends on model evaluation, data size, and convergence of the numerical refits. Changing only the uncertainty choice calls `refreshUncertaintyModel`, which clones the displayed function before recalculating its profile errors. The -original best-fit coefficients are preserved. The implementation reuses cached -temporary functions and the normal fit infrastructure; it is not an isolated, -side-effect-free statistical library. Tests cover the user-visible coefficient, +original best-fit coefficients are preserved. The implementation reuses cached temporary functions and the normal fitting +code, which also refreshes parts of the display. It is not a separate numerical +routine that only returns an array of errors. Tests cover the user-visible coefficient, Autofit, fixed-parameter, and selection behavior. ## 6. Rank, fixed parameters, and perfect fits @@ -200,16 +243,19 @@ functions use central differences with step `1e-5*max(1,abs(parameter))`. This is separate from the adaptive profile step above. For nonlinear models, the rank is local and does not establish global identifiability. -For `y=(A+B)*x`, the two derivative columns are identical. Four observations +A concrete example is `y=(A+B)*x`: the data can determine A+B, but cannot +separately determine A and B. Increasing either coefficient by the same amount +has the same effect on every predicted value, so their derivative columns are +identical. Four observations have p=2, r=1, and df=3. Refitting B can compensate for a perturbation in A: -individual A and B errors should be unavailable. Rank drives df and the report's -warning; each coefficient error still depends on its own profile-curvature -check. The method does not replace a redundant fit with a uniquely identified -parameterization. +individual A and B errors should be unavailable. The rank determines df and whether the report warns about dependent parameters. +Each coefficient error still depends on its own curvature check. The software +does not rewrite this function as a one-parameter fit to A+B. A fixed coefficient reports N/A, not an estimated error of zero: its value is an input constraint, and uncertainty in that input has not been propagated. -Manual mode likewise does not reuse automatic-fit inference. The current +In manual mode, the software likewise does not report errors left over from +an earlier automatic fit. The current implementation withholds parameter-error estimates when df<=0 even if a supplied noise model could support inference in a different implementation. @@ -233,15 +279,17 @@ Q is deliberately unavailable because the scale was learned from those residuals For small samples, estimated standard errors are not automatically normal-based confidence intervals; a regular linear model with unknown variance ordinarily -uses Student-t factors for coefficient intervals. For nonlinear, bounded, -weakly identified, or multimodal problems, a symmetric local curvature can be -misleading. Neither symmetric/asymmetric confidence intervals nor parameter +uses Student-t factors for coefficient intervals. A symmetric local error can also be misleading when the function depends +nonlinearly on its coefficients, a coefficient is near a physical boundary, +the data barely determine it, or several different coefficient sets fit well. Neither symmetric/asymmetric confidence intervals nor parameter p-values are added by this PR. -Numerical limitations include finite-difference cancellation, parameter scaling -(the additive 1 in the step rule is unit dependent), local minima, incomplete -refits, and strongly nonquadratic profiles. The accepted-D guard is not a -certificate of optimizer convergence. Very small/large uncertainty scales can +The numerical calculation has several limitations. Subtracting nearly equal +chi-square values can lose precision. Changing coefficient units affects the +step rule because it adds the number 1 to the coefficient magnitude. A refit +can stop short of its minimum or settle in a different local minimum. A profile +can depart substantially from a parabola. Passing the D threshold does not +establish that these problems were avoided. Very small/large uncertainty scales can also underflow/overflow when squared. Existing tests exercise selected cases, not all possible UserFunctions or parameterizations. diff --git a/test/fit-uncertainty-guide.md b/test/fit-uncertainty-guide.md index cbc8f7b8..5a923324 100644 --- a/test/fit-uncertainty-guide.md +++ b/test/fit-uncertainty-guide.md @@ -1,36 +1,50 @@ # Understanding fit uncertainties -A fit gives a best estimate and an uncertainty. They answer different questions: -**what value fits these measurements best, and how precisely do the measurements -constrain it?** A small uncertainty does not establish that the model or the -measurements are correct. +Suppose Tracker reports a velocity of **2.000 +/- 0.063 m/s**. -This guide describes the Data Tool behavior in OSP PR #9. The -[technical note](bevington-fit-uncertainties.md) explains the implementation. +- **2.000 m/s** is the velocity calculated from the fitted line's slope. +- **0.063 m/s** is the estimated uncertainty in that velocity. It estimates the + typical variation in fitted velocity if you repeated the measurements under + the same conditions. This estimate depends on the assumptions used in the fit. -## What the error beside a coefficient means +The number after `+/-` is an estimate of uncertainty, not a known mistake in the +velocity. Tracker does not know the object's true velocity. A small uncertainty +can accompany an inaccurate result if, for example, the distance calibration is +wrong. -Suppose a line describes position versus time: +This guide explains where those two numbers come from in OSP PR #9. The +[technical note](bevington-fit-uncertainties.md) describes the calculation in detail. + +## How Tracker finds the slope and its uncertainty + +For an object moving at constant velocity, position versus time follows a line: ```text x = A*t + B ``` -The slope A is velocity; B is position at t=0. To investigate uncertainty in A, -Tracker moves the slope slightly away from its best value, allows B to adjust, -and checks how much worse the fit becomes. It repeats this on the other side. -If the fit becomes worse quickly, the data constrain A tightly. If many nearby -slopes fit almost as well, its uncertainty is larger. This is the idea behind -the profile/refit curvature method identified in the code with Bevington's -Eq. 8.13. +A is the slope, which gives velocity. B is the position at t=0. Tracker chooses +A and B to make the sum of squared differences between the measured positions +and the line as small as possible. Those differences are called **residuals**. + +Finding the uncertainty takes another calculation. Tracker tries a slightly +steeper line and adjusts B to find the best position for that line. It then tries +a slightly shallower line. If even a small slope change makes the residuals much +larger, the slope uncertainty is small. If a wider range of slopes fits nearly +as well, the uncertainty is larger. The calculation also needs an estimate of +how uncertain the position measurements are. + +There are therefore two uncertainties to keep separate: uncertainty in **each +measured position**, and uncertainty in the **velocity calculated from the fit**. +They even have different units: metres and metres per second. -The reported error is a **standard-error estimate**, not a maximum possible -error and not a 95% confidence interval. Its interpretation depends on the -measurement-error model and the adequacy of the fitted function. +Tracker calls a fitted coefficient's uncertainty its **standard error**. It is +not a maximum possible error or a 95% confidence interval. The technical note +explains the method identified in the code with Bevington's Eq. 8.13. -## A small example you can reproduce in a spreadsheet +## An example you can reproduce in a spreadsheet -These are illustrative measurements, not a Tracker video: +Consider these illustrative position measurements: | Time (s) | Position (m) | |---|---| @@ -39,93 +53,94 @@ These are illustrative measurements, not a Tracker video: | 2 | 4.9 | | 3 | 7.1 | -The best line is `x = 2*t + 1`. The measured positions differ from it by -`+0.1, -0.1, -0.1, +0.1 m`, so the sum of squared residuals (SSE) is -`0.040 m^2`. +The fitted line is `x = 2*t + 1`. Its residuals are +`+0.1, -0.1, -0.1, +0.1 m`. Squaring and adding them gives the **sum of squared +residuals**, or SSE: `0.040 m^2`. -With four observations and two independent fitted parameters, the residual -standard error is `sqrt(0.040/2) = 0.141421... m`. Using that estimated scatter, -the coefficient results are approximately: +With four measurements and two independent fitted coefficients, the estimate +of position scatter is `sqrt(0.040/(4-2)) = 0.141421... m`. This quantity is +called the **residual standard error**. Using it as the uncertainty of each +position measurement gives: ```text -Velocity A 2.000 +/- 0.063 m/s -Position B 1.00 +/- 0.12 m +Velocity A 2.000 +/- 0.063 m/s +Position at t=0 B 1.00 +/- 0.12 m ``` -The velocity uncertainty is not the same as the scatter of individual position -measurements. It depends on the measurement times as well as the position -scatter. A longer time span can constrain a slope more tightly, provided the -same model still applies. +The slope uncertainty depends on when the positions were measured as well as +how scattered they are. Spreading measurements over a longer time can improve +the velocity estimate, provided the velocity remains constant. -## Where the uncertainty scale comes from +## Choosing the position uncertainty **Unweighted - estimate from residuals** is the default. All selected points -receive equal weight, and their scatter about the fit estimates the common -position uncertainty. This is a useful starting point when no independent -measurement uncertainty has been supplied. The usual residual-scatter estimate -is described in the [NIST least-squares guide](https://itl.nist.gov/div898/handbook/pmd/section4/pmd431.htm). - -**Known uncertainty (data units)** uses the positive uncertainty you supply. -For the same example, specifying `0.20 m` leaves the best line unchanged, but -makes the velocity uncertainty approximately `0.089 m/s`. The chosen uncertainty -scale has increased; the measured points have not changed. - -**Known uncertainty (pixels)** supplies the same kind of model through Tracker's -position calibration. If the calibration is `0.0025 m/pixel`, a choice of -`1.0 pixel` means `0.0025 m`; `0.5 pixel` means `0.00125 m`. One pixel is a -convenient starting choice, not an assertion about your marking accuracy. -The current conversion does not include uncertainty in the calibration itself. -Pixel mode is available only for eligible directly calibrated position data. - -## What agreement does—and does not—tell you - -Careless marking can increase the residual scatter and therefore increase -residual-estimated parameter errors. A result can then agree with an expected -value within its uncertainty because it is imprecise. A small discrepancy in -units of the reported error (often called a Z-value) does not, by itself, -establish good measurement technique. - -The full report offers two different checks: - -- **R-squared** describes how much of the observed variation the fitted model - accounts for. It does not establish that residuals are small relative to your - measurement uncertainty. -- **Reduced chi-square** compares residuals with the supplied uncertainty scale. - When Tracker estimates that scale from the same residuals, reduced chi-square - is 1 by construction whenever defined. It is then not an independent test, - and the chi-square probability Q is N/A. - -With a genuinely supplied common uncertainty, a reduced chi-square much above -one can indicate underestimated uncertainty, a poor model, or other violated -assumptions. A value well below one can indicate an overestimate or correlations. -These are reasons to inspect the experiment, not automatic diagnoses. - -## Position fits can give motion results directly - -For a line fit to position versus time, the slope and its error are velocity -and its error. For a quadratic `y = A*t^2 + B*t + C`, acceleration is `2*A` and -its standard error is `2*error(A)`. - -For example, if `A = -4.989 +/- 0.023 m/s^2`, the acceleration is -`-9.978 +/- 0.046 m/s^2`. The actual software uses unrounded values before -formatting the result. B gives velocity at the data's t=0, which need not be the -first selected frame. These derived quantities appear in copied reports when -Tracker identifies the columns as position versus time. - -The pixel setting does not propagate uncertainties into the separately -calculated velocity/acceleration columns. Those calculations reuse positions, -so their errors can be correlated. For this purpose, fitting the measured -position data avoids a second regression on differentiated data. - -## Reporting a result - -Keep the full precision for subsequent calculations; round at the reporting -stage. Tracker displays two significant digits in positive uncertainty values -and matches the coefficient's displayed decimal place to its uncertainty. -Copied numeric results retain full precision. - -Record the fitted interval, equation, units, and uncertainty choice with the -result. If an error is N/A, it is unavailable—not zero. Manual parameters, -fixed coefficients, insufficient information, and unidentifiable parameters -require particular care. The full report is available from **... → Copy Full -Fit Report** when you need the details. +receive equal weight. Tracker uses their scatter about the fitted line to +estimate their common measurement uncertainty, as in the example above. +This is useful when you have not estimated that uncertainty independently. +The [NIST least-squares guide](https://itl.nist.gov/div898/handbook/pmd/section4/pmd431.htm) +describes this residual-scatter estimate. + +**Known uncertainty (data units)** lets you supply that estimate yourself. +For example, entering `0.20 m` means you are assigning that standard uncertainty +to each position measurement. The example's fitted velocity stays at +`2.000 m/s`, but its standard error increases from about `0.063` to `0.089 m/s`. +The line stays the same because every point still has equal weight. + +**Known uncertainty (pixels)** lets you describe marking precision in image +pixels. With a calibration of `0.0025 m/pixel`, choosing `1.0 pixel` assigns a +position uncertainty of `0.0025 m`. Choosing `0.5 pixel` assigns `0.00125 m`. +One pixel is a starting choice; you must judge whether it describes your marking. +This option requires suitable position calibration and does not include +uncertainty in the calibration itself. + +## Why a result can agree with expectations despite poor measurements + +Careless marking can produce more scatter. In the default mode, more scatter +can produce larger errors on the fitted coefficients. A measured result may +then lie within one reported error of the expected value because that error is +large. Agreement and precision are separate things to examine. + +The full report includes two useful statistics: + +- **R-squared** describes how much of the observed variation the fit accounts + for. A value near one does not establish accurate measurement or a correct + model. +- **Reduced chi-square** compares the residuals with the measurement uncertainty. + With an independently supplied uncertainty, a value well above one means + more scatter than expected; a value well below one means less. Several + causes are possible, so inspect the measurements and model before drawing + conclusions. + +When the measurement uncertainty is estimated from these same residuals, +reduced chi-square is exactly one whenever the calculation is defined. That +value cannot independently confirm the fit. The chi-square probability Q is +therefore shown as N/A in this mode. + +## Getting velocity and acceleration from position fits + +For a line fit, velocity is A and its standard error is the error reported for A. +For a quadratic `y = A*t^2 + B*t + C`, acceleration is `2*A`; its standard error +is twice the error reported for A. + +For example, `A = -4.989 +/- 0.023 m/s^2` gives an acceleration of +`-9.978 +/- 0.046 m/s^2`. B gives velocity at t=0, which need not be the first +selected frame. Tracker includes these results in copied reports when it +identifies the data as position versus time. + +This calculation uses the position fit. The pixel setting does not assign +errors to Tracker's separate velocity and acceleration data columns. Those +columns combine measurements from several frames, and their errors can be +related because they reuse the same positions. + +## Writing the result in a lab report + +Keep full precision during calculations and round when writing the final result. +Tracker displays two significant digits in a positive uncertainty and rounds +the fitted value to the same decimal place. Copied numeric results retain their +full precision. + +Record the selected time interval, equation, units, and uncertainty choice. +N/A means that an error estimate is unavailable. This can happen when you enter +parameters manually, hold a coefficient fixed, or provide too little information +to determine a coefficient's uncertainty. Use **... → Copy Full Fit Report** +when you need the additional statistics. From f6b9a34341c5295ae14d72d05f39d3aba7ba7e03 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 11:33:50 -0400 Subject: [PATCH 14/21] Show intermediate steps in the profile uncertainty example --- test/bevington-fit-uncertainties.md | 168 +++++++++++++++++++++++++--- 1 file changed, 152 insertions(+), 16 deletions(-) diff --git a/test/bevington-fit-uncertainties.md b/test/bevington-fit-uncertainties.md index 1e3340b6..ae6272be 100644 --- a/test/bevington-fit-uncertainties.md +++ b/test/bevington-fit-uncertainties.md @@ -161,29 +161,165 @@ parameter errors. ## 4. A reproducible analytic example Take `x=[0,1,2,3]`, `y=[1.1,2.9,4.9,7.1]` and fit `y=A*x+B`. -The exact least-squares coefficients are A=2 and B=1, SSE=0.04, n=4, r=2, -df=2, and estimated sigma_y=sqrt(0.02). +The following steps show where the numerical constants come from. -Here `x_mean=1.5` and `Sxx=sum((x-x_mean)^2)=5`. At a perturbed slope `2+h`, -the refitted intercept is `1-1.5*h`. Consequently, +### Start with the fitted line and residual variance + +The fitted coefficients are A=2 and B=1. Check the residuals directly: + +| x | Measured y | Predicted y=2*x+1 | Residual e=measured-predicted | +|---|---|---|---| +| 0 | 1.1 | 1.0 | +0.1 | +| 1 | 2.9 | 3.0 | -0.1 | +| 2 | 4.9 | 5.0 | -0.1 | +| 3 | 7.1 | 7.0 | +0.1 | ```text +SSE_min = 0.1^2 + (-0.1)^2 + (-0.1)^2 + 0.1^2 = 0.04 +n = 4 observations +r = 2 independent fitted coefficients +df = n-r = 4-2 = 2 +sigma_y^2 = SSE_min/df = 0.04/2 = 0.02 +``` + +**0.02 is the estimated variance**, not the standard deviation. The standard +deviation is `sigma_y=sqrt(0.02)=0.141421...`. Chi-square divides SSE by the +variance, so the minimum chi-square is `0.04/0.02=2`. + +### Change the slope and refit the intercept + +Let h be a change in slope, so the trial slope is `A=2+h`. The intercept must +be refitted at each trial slope. A least-squares line with a free intercept +passes through the mean point `(x_mean,y_mean)`: + +```text +x_mean = (0+1+2+3)/4 = 1.5 +y_mean = (1.1+2.9+4.9+7.1)/4 = 4 +B = y_mean - A*x_mean + = 4 - (2+h)*1.5 + = 1 - 1.5*h +``` + +The trial line's prediction therefore differs from the original line by +`h*(x-1.5)`. Each new residual is `e-h*(x-1.5)`, where e is the original +residual in the table. Square those new residuals and add them: + +```text +SSE_profile(A) = sum([e-h*(x-1.5)]^2) + = sum(e^2) - 2*h*sum(e*(x-1.5)) + h^2*sum((x-1.5)^2) +``` + +The middle sum is zero: + +```text +sum(e*(x-1.5)) = 0.1*(-1.5) + (-0.1)*(-0.5) + + (-0.1)*0.5 + 0.1*1.5 + = -0.15 + 0.05 - 0.05 + 0.15 = 0 +``` + +The last sum, called Sxx, gives the **5**: + +```text +Sxx = sum((x-x_mean)^2) + = (-1.5)^2 + (-0.5)^2 + 0.5^2 + 1.5^2 + = 2.25 + 0.25 + 0.25 + 2.25 = 5 + SSE_profile(A) = 0.04 + 5*h^2 -chi^2_profile(A) = 2 + 250*h^2 -D = 500*delta^2 -sigma_A = sqrt(0.02/5) = 0.0632455532033676 -sigma_B = sqrt(0.02*(1/4 + 1.5^2/5)) = 0.1183215956619923 ``` -Holding B fixed instead would use sum(x^2)=14 and give the smaller conditional -slope error `sqrt(0.02/14)=0.0377964473`. That is why the refit matters. +### Divide by the variance: where 250 comes from + +Keep the estimated variance at 0.02 while testing the trial slopes: + +```text +chi^2_profile(A) = SSE_profile(A)/sigma_y^2 + = (0.04 + 5*h^2)/0.02 + = 0.04/0.02 + (5/0.02)*h^2 + = 2 + 250*h^2 +``` + +Thus **250 is 5 divided by 0.02**. It describes how quickly chi-square increases +as the slope moves away from 2, after the intercept has been refitted. + +### Add the increases on both sides: where 500 comes from + +The code tests two slope changes: `h=-delta` and `h=+delta`. Squaring removes +the sign, so both trials have the same increase in this example: + +```text +chi^2_minus = 2 + 250*delta^2 +chi^2_plus = 2 + 250*delta^2 +chi^2_min = 2 + +D = chi^2_minus + chi^2_plus - 2*chi^2_min + = (2 + 250*delta^2) + (2 + 250*delta^2) - 2*2 + = 500*delta^2 +``` + +**500 is 250+250**, because D includes both increases. As a numerical check, +choose `delta=0.01`: each trial has chi-square 2.025, and +`D=2.025+2.025-4=0.05`. This is also `500*(0.01)^2`. + +### Convert that increase into the slope uncertainty + +Substitute D into the curvature formula from section 2. Since delta is a +positive step size, it cancels: + +```text +sigma_A = delta*sqrt(2/D) + = delta*sqrt(2/(500*delta^2)) + = sqrt(2/500) + = sqrt(0.004) + = sqrt(0.02/5) + = 0.0632455532033676 +``` + +Using the numerical trial above gives the same answer: +`0.01*sqrt(2/0.05)=0.0632455532033676`. This cancellation is exact for this +quadratic profile; a numerical nonlinear fit need not behave so simply. + +### Calculate the intercept uncertainty + +For a straight line, profiling the intercept while refitting the slope gives +the familiar expression below. Its inputs have all been calculated above: +the variance is 0.02, n is 4, x_mean is 1.5, and Sxx is 5. + +```text +sigma_B^2 = sigma_y^2*(1/n + x_mean^2/Sxx) + = 0.02*(1/4 + 1.5^2/5) + = 0.02*(0.25 + 2.25/5) + = 0.02*(0.25 + 0.45) + = 0.014 + +sigma_B = sqrt(0.014) = 0.1183215956619923 +``` + +The `1/n` term describes uncertainty in the fitted line's height at x_mean. +The `x_mean^2/Sxx` term adds the effect of uncertainty in the slope when finding +the intercept at x=0. They combine to give the intercept variance. + +### Compare with holding the intercept fixed + +If B were held at 1 rather than refitted, changing A would change the prediction +by `h*x` instead of `h*(x-x_mean)`. The squared-residual increase would then +use `sum(x^2)=0+1+4+9=14` instead of Sxx=5. The resulting slope error would be +`sqrt(0.02/14)=0.0377964473`. It is smaller because B is being treated as known +exactly. That is a different assumption from estimating both A and B. + +### Change the supplied measurement uncertainty + +If the independently supplied sigma_y is 0.20 instead, its variance is 0.04. +A and B stay unchanged. Substituting this variance gives +`sigma_A=sqrt(0.04/5)=0.0894427191` and +`sigma_B=sqrt(0.04*0.70)=0.1673320053`. -If the independently supplied sigma_y is 0.20 instead, A and B are unchanged, -but `sigma_A=0.0894427191`, `sigma_B=0.1673320053`, chi-square=1, -reduced chi-square=0.5, and Q=exp(-1/2)=0.6065306597 for df=2. -With the residual-estimated scale, chi-square=2, reduced chi-square=1, -and Q is N/A. More generally, scaling a supplied sigma_y by c scales regular -profile errors by |c| and divides chi-square by c^2, up to numerical error. +Chi-square is now `SSE_min/sigma_y^2=0.04/0.04=1`, and reduced chi-square is +`1/df=1/2=0.5`. For df=2, the survival probability is +`Q=exp(-chi^2/2)=exp(-1/2)=0.6065306597`. +With the residual-estimated variance of 0.02, chi-square is 2, reduced +chi-square is 1, and Q is N/A. More generally, multiplying a supplied sigma_y +by a positive factor c multiplies regular profile errors by c and divides +chi-square by c^2, up to numerical error. ## 5. Code path and numerical procedure From 5adff93a4a0e32ebe6bdfc2773d25c16d934df48 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 11:38:58 -0400 Subject: [PATCH 15/21] Keep residual variance explicit in the uncertainty example --- test/bevington-fit-uncertainties.md | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/bevington-fit-uncertainties.md b/test/bevington-fit-uncertainties.md index ae6272be..d6f9abe1 100644 --- a/test/bevington-fit-uncertainties.md +++ b/test/bevington-fit-uncertainties.md @@ -161,7 +161,7 @@ parameter errors. ## 4. A reproducible analytic example Take `x=[0,1,2,3]`, `y=[1.1,2.9,4.9,7.1]` and fit `y=A*x+B`. -The following steps show where the numerical constants come from. +The following steps trace the residuals through to the coefficient uncertainties. ### Start with the fitted line and residual variance @@ -227,53 +227,53 @@ Sxx = sum((x-x_mean)^2) SSE_profile(A) = 0.04 + 5*h^2 ``` -### Divide by the variance: where 250 comes from +### Divide by the residual variance -Keep the estimated variance at 0.02 while testing the trial slopes: +Keep the estimated variance at 0.02 while testing the trial slopes. To convert +SSE to chi-square, divide by that variance: ```text chi^2_profile(A) = SSE_profile(A)/sigma_y^2 = (0.04 + 5*h^2)/0.02 - = 0.04/0.02 + (5/0.02)*h^2 - = 2 + 250*h^2 + = 2 + (5*h^2)/0.02 ``` -Thus **250 is 5 divided by 0.02**. It describes how quickly chi-square increases -as the slope moves away from 2, after the intercept has been refitted. +The increase above the minimum is therefore `(5*h^2)/0.02`: the increase in +SSE divided by the residual variance. -### Add the increases on both sides: where 500 comes from +### Add the increases on both sides The code tests two slope changes: `h=-delta` and `h=+delta`. Squaring removes the sign, so both trials have the same increase in this example: ```text -chi^2_minus = 2 + 250*delta^2 -chi^2_plus = 2 + 250*delta^2 +chi^2_minus = 2 + (5*delta^2)/0.02 +chi^2_plus = 2 + (5*delta^2)/0.02 chi^2_min = 2 D = chi^2_minus + chi^2_plus - 2*chi^2_min - = (2 + 250*delta^2) + (2 + 250*delta^2) - 2*2 - = 500*delta^2 + = [2 + (5*delta^2)/0.02] + [2 + (5*delta^2)/0.02] - 2*2 + = (2*5*delta^2)/0.02 ``` -**500 is 250+250**, because D includes both increases. As a numerical check, -choose `delta=0.01`: each trial has chi-square 2.025, and -`D=2.025+2.025-4=0.05`. This is also `500*(0.01)^2`. +The factor of 2 counts the two equal increases. As a numerical check, choose +`delta=0.01`. Each increase is `5*(0.01)^2/0.02=0.025`, so each trial has +chi-square 2.025 and `D=2.025+2.025-4=0.05`. ### Convert that increase into the slope uncertainty -Substitute D into the curvature formula from section 2. Since delta is a -positive step size, it cancels: +Substitute D into the curvature formula from section 2. The factor of 2 +cancels, and since delta is a positive step size, delta cancels too: ```text sigma_A = delta*sqrt(2/D) - = delta*sqrt(2/(500*delta^2)) - = sqrt(2/500) - = sqrt(0.004) + = delta*sqrt(2/[(2*5*delta^2)/0.02]) + = delta*sqrt(0.02/(5*delta^2)) = sqrt(0.02/5) = 0.0632455532033676 ``` +The result is the square root of the residual variance divided by Sxx. Using the numerical trial above gives the same answer: `0.01*sqrt(2/0.05)=0.0632455532033676`. This cancellation is exact for this quadratic profile; a numerical nonlinear fit need not behave so simply. From 50d214a6cfce2efbbe118138187f48ff4c0c86ff Mon Sep 17 00:00:00 2001 From: paulnord Date: Mon, 7 Sep 2026 10:52:27 -0500 Subject: [PATCH 16/21] Split fit uncertainty docs by audience --- test/fit-uncertainty-details.md | 374 ++++++++++++++++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 test/fit-uncertainty-details.md diff --git a/test/fit-uncertainty-details.md b/test/fit-uncertainty-details.md new file mode 100644 index 00000000..70094549 --- /dev/null +++ b/test/fit-uncertainty-details.md @@ -0,0 +1,374 @@ +# Understanding fit uncertainties: the statistics behind the result + +> **A number from a fit is not the answer; it is a measurement with assumptions attached.** + +This is the middle layer of the Tracker fit-uncertainty documentation. + +- **Just using Tracker for a lab?** Start with the [short guide](fit-uncertainty-guide.md). +- **Want to know where the uncertainty comes from?** Read this document. +- **Reviewing or maintaining the implementation?** See the [Bevington-style implementation note](bevington-fit-uncertainties.md). + +This note assumes that you already know what a residual is and have seen least-squares fitting before. It develops the statistical ideas behind Tracker's reported coefficient standard errors without following the Java code line by line. + +## 1. From residuals to a measurement scale + +Suppose Tracker fits a function `f(x)` to measured values `y_i`. The residual for point `i` is + +```text +residual_i = y_i - f(x_i) +``` + +and the sum of squared residuals is + +```text +SSE = sum(residual_i^2). +``` + +Tracker currently treats all selected observations as having the same y uncertainty. There are two main ways to set that common uncertainty. + +### Estimate it from the residuals + +If no independent measurement uncertainty is supplied, Tracker estimates a common variance from the scatter about the fitted curve: + +```text +sigma_y^2 = SSE_min / df +``` + +where `df` is the residual degrees of freedom. For an ordinary full-rank fit with `n` observations and `p` independently determined fitted coefficients, + +```text +df = n - p. +``` + +More generally Tracker uses the numerical rank `r` of the free parameter directions, so + +```text +df = n - r. +``` + +This distinction matters when two adjustable coefficients cannot actually be determined independently from the data. + +Because this mode defines `sigma_y` from the fitted residuals themselves, + +```text +chi^2_min = SSE_min / sigma_y^2 = df +``` + +and therefore reduced chi-square is exactly one whenever the calculation is defined. It is one **by construction**, so it is not an independent goodness-of-fit test in this mode. + +### Supply the measurement uncertainty + +If an experiment provides an independent common uncertainty `sigma_y`, Tracker instead uses + +```text +chi^2 = SSE / sigma_y^2. +``` + +The best-fit coefficients do not change when the same positive uncertainty is assigned to every point: multiplying the objective by a constant does not move its minimum. The uncertainty scale and chi-square statistics do change. + +The pixel option is another way of supplying the same kind of common uncertainty. Tracker converts the selected pixel uncertainty to physical position units using the available calibration scale. It does not, in this calculation, propagate uncertainty in the ruler or calibration itself. + +## 2. Why moving a coefficient tells us its uncertainty + +Consider a straight-line fit + +```text +y = A*x + B. +``` + +The best-fit values of `A` and `B` occur at the minimum of the fit objective. To estimate the uncertainty in `A`, Tracker asks a practical question: + +> How far can I move `A` before the fit becomes noticeably worse, while still allowing the other fitted coefficients to adjust? + +Tracker therefore holds `A` at a nearby trial value, refits `B`, and calculates the resulting chi-square. Repeating this on both sides of the best-fit value traces the local **profile** of chi-square for `A`. + +Near a well-behaved minimum that profile is approximately quadratic: + +```text +chi^2_profile(A_hat + h) ~= chi^2_min + h^2 / sigma_A^2. +``` + +So a displacement of about one standard error corresponds locally to an increase + +```text +Delta chi^2 ~= 1. +``` + +Tracker estimates the local curvature using equal trial steps `delta` on either side: + +```text +D = chi^2_minus + chi^2_plus - 2*chi^2_min +sigma_A ~= delta * sqrt(2/D). +``` + +The factor of two appears because `D` contains the increases from **both** sides of the minimum. + +This is a symmetric local standard-error estimate. Tracker does not search independently for left and right confidence limits. + +## 3. Why the other coefficients are refitted + +Suppose the fitted model is still + +```text +y = A*x + B. +``` + +If the trial slope `A` becomes slightly larger, the intercept `B` can often shift to compensate. Holding `B` fixed would answer a different question: + +> How uncertain is `A` if `B` is already known exactly? + +Refitting `B` asks the experimentally relevant question: + +> How uncertain is `A` when both `A` and `B` must be inferred from these same data? + +This is why correlated fitted coefficients matter even when Tracker reports one standard error beside each coefficient. + +For a regular linear least-squares problem, the local profile-curvature result agrees with the usual covariance-matrix standard error when both calculations use the same measurement-uncertainty scale. The implementation intentionally retains the numerical profile/refit calculation. + +## 4. A reproducible straight-line example + +Take + +```text +x = [0, 1, 2, 3] +y = [1.1, 2.9, 4.9, 7.1] +``` + +and fit + +```text +y = A*x + B. +``` + +The fitted line is + +```text +A = 2 +B = 1. +``` + +The residuals are + +```text ++0.1, -0.1, -0.1, +0.1 +``` + +so + +```text +SSE_min = 0.04. +``` + +There are four observations and two independent fitted coefficients, giving + +```text +df = 4 - 2 = 2 +sigma_y^2 = 0.04 / 2 = 0.02 +sigma_y = sqrt(0.02) = 0.141421... +``` + +The value `0.02` is the **variance**; `0.141421...` is the standard deviation. + +### Profile the slope + +Let the trial slope be + +```text +A = 2 + h. +``` + +For every trial slope the intercept is refitted. A least-squares line with a free intercept passes through the mean point. Here + +```text +x_mean = 1.5 +y_mean = 4 +``` + +so + +```text +B = y_mean - A*x_mean + = 4 - (2+h)*1.5 + = 1 - 1.5*h. +``` + +The resulting profile SSE is + +```text +SSE_profile(A) = 0.04 + 5*h^2, +``` + +where + +```text +Sxx = sum((x-x_mean)^2) = 5. +``` + +Dividing by the fixed residual variance gives + +```text +chi^2_profile(A) + = (0.04 + 5*h^2)/0.02 + = 2 + 5*h^2/0.02. +``` + +The slope standard error is therefore + +```text +sigma_A = sqrt(0.02/5) + = 0.063245553... +``` + +so the fitted slope may be reported approximately as + +```text +A = 2.000 +/- 0.063. +``` + +### Profile the intercept + +The corresponding straight-line result for the intercept is + +```text +sigma_B^2 = sigma_y^2 * (1/n + x_mean^2/Sxx) + = 0.02 * (1/4 + 1.5^2/5) + = 0.014 + +sigma_B = 0.1183216... +``` + +The two coefficient uncertainties differ because slope and intercept affect the fitted line in different ways. + +### What if the intercept were known exactly? + +If `B` were held fixed at 1 instead of being refitted, the slope uncertainty would use `sum(x^2)=14` rather than `Sxx=5`: + +```text +sigma_A_fixed_B = sqrt(0.02/14) + = 0.0377964... +``` + +That smaller number is not a better estimate of the same problem. It answers a different problem in which the intercept is assumed known exactly. + +## 5. Supplying a different measurement uncertainty + +Suppose the same four measurements have an independently justified common uncertainty + +```text +sigma_y = 0.20. +``` + +Then + +```text +sigma_y^2 = 0.04. +``` + +The best-fit line remains `A=2`, `B=1`, because every point still has equal weight. The coefficient uncertainties scale with the supplied uncertainty: + +```text +sigma_A = sqrt(0.04/5) = 0.0894427... +sigma_B = sqrt(0.04*0.70) = 0.1673320... +``` + +The fit statistics now have independent content: + +```text +chi^2 = SSE_min/sigma_y^2 = 1 +reduced chi^2 = 1/2 = 0.5. +``` + +For `df=2`, the chi-square survival probability is about `Q=0.607`. `Q` is the probability, assuming the model and uncertainty assumptions, of obtaining a chi-square at least this large from repeated data. It is **not** the probability that the model is true. + +## 6. When two adjustable coefficients are not two measurements + +Consider + +```text +y = (A+B)*x. +``` + +The data can determine `A+B`, but they cannot determine `A` and `B` separately. Increasing `A` while decreasing `B` by the same amount leaves every prediction unchanged. + +So two editable coefficients do not necessarily represent two independently measurable parameter directions. Tracker estimates the local numerical rank `r` of those directions and uses + +```text +df = n - r. +``` + +For four observations of this model, + +```text +p = 2 editable coefficients +r = 1 independent direction +df = 3. +``` + +Tracker reports a warning for dependent parameters and does not pretend that the individual `A` and `B` uncertainties are meaningful. + +The exact numerical rank algorithm is described in the implementation note. + +## 7. What this standard error does not include + +Tracker's current fit uncertainty model assumes that: + +- the x values are known well enough to treat as exact; +- residual measurement errors are independent; +- selected y measurements share a common uncertainty; +- the fitted model is an adequate local description of the data; +- the profile is sufficiently regular near its minimum for a symmetric local error to be useful. + +It does **not** automatically include uncertainty from: + +- ruler or spatial calibration; +- frame timing; +- lens distortion; +- motion outside the calibration plane; +- an inadequate physical model; +- correlations in derived velocity or acceleration columns. + +A small reported coefficient uncertainty therefore does not guarantee an accurate physical result. Conversely, a large uncertainty may make an inaccurate-looking result statistically compatible with an expected value simply because the experiment was imprecise. + +Residual plots and experimental choices still matter. + +## 8. From fitted coefficients to motion quantities + +For a line fit + +```text +x = A*t + B, +``` + +velocity is `A`, so its standard error is `sigma_A`. + +For a quadratic position fit + +```text +y = A*t^2 + B*t + C, +``` + +acceleration is + +```text +a = 2*A +``` + +with standard error + +```text +sigma_a = 2*sigma_A. +``` + +The coefficient `B` is velocity at `t=0`. Velocity at another time, + +```text +v(t) = 2*A*t + B, +``` + +depends on both `A` and `B`. Its uncertainty generally requires their covariance or an equivalent reparameterized/profile calculation. Tracker does not obtain that uncertainty by pretending the two coefficient errors are independent. + +## Where to go next + +If this is enough detail, return to the [short guide](fit-uncertainty-guide.md) and use the fit thoughtfully. + +If you want the exact numerical step-size rules, Java code path, rank calculation, perfect-fit behavior, and regression tests, continue to the [Bevington-style implementation note](bevington-fit-uncertainties.md). From 7b8511ded2ae2dcd7a0e994991b268ae86bd3946 Mon Sep 17 00:00:00 2001 From: paulnord Date: Mon, 7 Sep 2026 10:53:10 -0500 Subject: [PATCH 17/21] Refocus fit uncertainty guide for students --- test/fit-uncertainty-guide.md | 262 +++++++++++++++++++++------------- 1 file changed, 165 insertions(+), 97 deletions(-) diff --git a/test/fit-uncertainty-guide.md b/test/fit-uncertainty-guide.md index 5a923324..7cc24d64 100644 --- a/test/fit-uncertainty-guide.md +++ b/test/fit-uncertainty-guide.md @@ -1,19 +1,18 @@ # Understanding fit uncertainties +> **A number from a fit is not the answer; it is a measurement with assumptions attached.** + Suppose Tracker reports a velocity of **2.000 +/- 0.063 m/s**. - **2.000 m/s** is the velocity calculated from the fitted line's slope. -- **0.063 m/s** is the estimated uncertainty in that velocity. It estimates the - typical variation in fitted velocity if you repeated the measurements under - the same conditions. This estimate depends on the assumptions used in the fit. +- **0.063 m/s** is Tracker's estimate of how precisely that velocity was determined from the selected measurements and uncertainty model. + +The number after `+/-` is not a known mistake in the velocity. Tracker does not know the object's true velocity. A small uncertainty can accompany an inaccurate result if, for example, the distance calibration is wrong. -The number after `+/-` is an estimate of uncertainty, not a known mistake in the -velocity. Tracker does not know the object's true velocity. A small uncertainty -can accompany an inaccurate result if, for example, the distance calibration is -wrong. +This is the short guide for using fit uncertainties in a lab. -This guide explains where those two numbers come from in OSP PR #9. The -[technical note](bevington-fit-uncertainties.md) describes the calculation in detail. +- **Want the statistical reasoning and a worked example?** Continue to [the details](fit-uncertainty-details.md). +- **Reviewing the numerical implementation?** See the [Bevington-style implementation note](bevington-fit-uncertainties.md). ## How Tracker finds the slope and its uncertainty @@ -23,28 +22,24 @@ For an object moving at constant velocity, position versus time follows a line: x = A*t + B ``` -A is the slope, which gives velocity. B is the position at t=0. Tracker chooses -A and B to make the sum of squared differences between the measured positions -and the line as small as possible. Those differences are called **residuals**. +`A` is the slope, which gives velocity. `B` is the position at `t=0`. + +Tracker chooses `A` and `B` to make the measured positions lie as close as possible to the fitted line. The vertical differences between the measurements and the line are called **residuals**. + +To estimate the uncertainty in `A`, Tracker tries a slightly steeper line and lets `B` readjust to give the best possible fit. It also tries a slightly shallower line. If even a small change in slope makes the fit much worse, the slope is tightly determined. If many nearby slopes fit almost equally well, the slope uncertainty is larger. + +There are two uncertainties to keep separate: -Finding the uncertainty takes another calculation. Tracker tries a slightly -steeper line and adjusts B to find the best position for that line. It then tries -a slightly shallower line. If even a small slope change makes the residuals much -larger, the slope uncertainty is small. If a wider range of slopes fits nearly -as well, the uncertainty is larger. The calculation also needs an estimate of -how uncertain the position measurements are. +- uncertainty in **each measured position**; +- uncertainty in the **velocity calculated from all of those positions**. -There are therefore two uncertainties to keep separate: uncertainty in **each -measured position**, and uncertainty in the **velocity calculated from the fit**. They even have different units: metres and metres per second. -Tracker calls a fitted coefficient's uncertainty its **standard error**. It is -not a maximum possible error or a 95% confidence interval. The technical note -explains the method identified in the code with Bevington's Eq. 8.13. +Tracker calls a fitted coefficient's uncertainty its **standard error**. It is not a maximum possible error and it is not automatically a 95% confidence interval. -## An example you can reproduce in a spreadsheet +## A small example -Consider these illustrative position measurements: +Suppose four position measurements are | Time (s) | Position (m) | |---|---| @@ -53,94 +48,167 @@ Consider these illustrative position measurements: | 2 | 4.9 | | 3 | 7.1 | -The fitted line is `x = 2*t + 1`. Its residuals are -`+0.1, -0.1, -0.1, +0.1 m`. Squaring and adding them gives the **sum of squared -residuals**, or SSE: `0.040 m^2`. +The fitted line is -With four measurements and two independent fitted coefficients, the estimate -of position scatter is `sqrt(0.040/(4-2)) = 0.141421... m`. This quantity is -called the **residual standard error**. Using it as the uncertainty of each -position measurement gives: +```text +x = 2*t + 1 +``` + +and Tracker reports approximately ```text Velocity A 2.000 +/- 0.063 m/s Position at t=0 B 1.00 +/- 0.12 m ``` -The slope uncertainty depends on when the positions were measured as well as -how scattered they are. Spreading measurements over a longer time can improve -the velocity estimate, provided the velocity remains constant. +The points do not fall exactly on the line. Their scatter helps determine how precisely the slope and intercept are known. + +The full calculation, including residuals, degrees of freedom, and why the slope uncertainty is `0.063 m/s`, is worked out in [the statistics guide](fit-uncertainty-details.md). ## Choosing the position uncertainty -**Unweighted - estimate from residuals** is the default. All selected points -receive equal weight. Tracker uses their scatter about the fitted line to -estimate their common measurement uncertainty, as in the example above. -This is useful when you have not estimated that uncertainty independently. -The [NIST least-squares guide](https://itl.nist.gov/div898/handbook/pmd/section4/pmd431.htm) -describes this residual-scatter estimate. - -**Known uncertainty (data units)** lets you supply that estimate yourself. -For example, entering `0.20 m` means you are assigning that standard uncertainty -to each position measurement. The example's fitted velocity stays at -`2.000 m/s`, but its standard error increases from about `0.063` to `0.089 m/s`. -The line stays the same because every point still has equal weight. - -**Known uncertainty (pixels)** lets you describe marking precision in image -pixels. With a calibration of `0.0025 m/pixel`, choosing `1.0 pixel` assigns a -position uncertainty of `0.0025 m`. Choosing `0.5 pixel` assigns `0.00125 m`. -One pixel is a starting choice; you must judge whether it describes your marking. -This option requires suitable position calibration and does not include -uncertainty in the calibration itself. - -## Why a result can agree with expectations despite poor measurements - -Careless marking can produce more scatter. In the default mode, more scatter -can produce larger errors on the fitted coefficients. A measured result may -then lie within one reported error of the expected value because that error is -large. Agreement and precision are separate things to examine. - -The full report includes two useful statistics: - -- **R-squared** describes how much of the observed variation the fit accounts - for. A value near one does not establish accurate measurement or a correct - model. -- **Reduced chi-square** compares the residuals with the measurement uncertainty. - With an independently supplied uncertainty, a value well above one means - more scatter than expected; a value well below one means less. Several - causes are possible, so inspect the measurements and model before drawing - conclusions. - -When the measurement uncertainty is estimated from these same residuals, -reduced chi-square is exactly one whenever the calculation is defined. That -value cannot independently confirm the fit. The chi-square probability Q is -therefore shown as N/A in this mode. +Tracker currently gives every selected point the same position uncertainty. You can choose how that common uncertainty is obtained. + +### Unweighted - estimate from residuals + +This is the default. Tracker gives all selected points equal weight and estimates their common uncertainty from how much they scatter about the fitted curve. + +This is useful when you do not have an independent estimate of the measurement uncertainty. + +An important consequence is that **reduced chi-square is exactly 1 by construction** whenever this calculation is defined. That value cannot independently confirm that the model is good. + +### Known uncertainty (data units) + +Use this when you have an independent estimate of the uncertainty of each measured position. + +For example, entering `0.20 m` means that you are assigning a standard uncertainty of `0.20 m` to every selected position. Because every point still has the same weight, the best-fit line does not move. The reported coefficient uncertainties and chi-square statistics do change. + +### Known uncertainty (pixels) + +This describes marking precision in image pixels. + +For example, with a calibration of `0.0025 m/pixel`: + +```text +1.0 pixel -> 0.0025 m +0.5 pixel -> 0.00125 m +``` + +One pixel is a reasonable starting point, not a universal truth. You must judge how precisely the object can actually be marked. + +This setting describes marking uncertainty only. It does not include uncertainty in the ruler, camera geometry, timing, or calibration itself. + +## A precise result can still be wrong + +Suppose a class measures gravitational acceleration and obtains + +```text +g = 10.10 +/- 0.03 m/s^2. +``` + +The small uncertainty says the tracked points determine that fitted value very consistently. It does **not** say that the calibration was correct or that the physical model included every important effect. + +Calibration, timing, lens distortion, motion outside the calibration plane, or a poor model can produce a precise but inaccurate result. + +Likewise, careless marking can increase the scatter and make the reported uncertainty larger. An inaccurate result may then appear to agree with an expected value simply because the experiment was imprecise. + +**Precision and accuracy are different questions.** + +## What R-squared and reduced chi-square tell you + +Tracker's full fit report includes several statistics. Two are especially useful in introductory work. + +### R-squared + +R-squared describes how much of the observed variation the fitted curve accounts for. + +A value close to one means the curve follows the data closely. It does **not** prove that: + +- the physical model is correct; +- the calibration is correct; +- the result is accurate; +- the measurement uncertainty is realistic. + +A high R-squared is therefore useful, but it is not a certificate of experimental correctness. + +### Reduced chi-square + +When you supply an independent measurement uncertainty, reduced chi-square compares the observed residual scatter with that stated uncertainty. + +As a rough guide: + +- near 1: scatter is comparable to what the uncertainty model predicts; +- much greater than 1: more scatter than expected; +- much less than 1: less scatter than expected. + +Those outcomes can have several causes. Inspect the residuals, measurements, calibration, and model before drawing conclusions. + +In the default residual-estimated mode, reduced chi-square is 1 by construction, so Tracker does not treat it as an independent fit check. The chi-square probability `Q` is shown as N/A in that mode. + +The [statistics guide](fit-uncertainty-details.md) explains why. ## Getting velocity and acceleration from position fits -For a line fit, velocity is A and its standard error is the error reported for A. -For a quadratic `y = A*t^2 + B*t + C`, acceleration is `2*A`; its standard error -is twice the error reported for A. +For a line fit + +```text +x = A*t + B +``` + +velocity is `A`, and its standard error is the standard error reported for `A`. + +For a quadratic position fit + +```text +y = A*t^2 + B*t + C +``` + +acceleration is -For example, `A = -4.989 +/- 0.023 m/s^2` gives an acceleration of -`-9.978 +/- 0.046 m/s^2`. B gives velocity at t=0, which need not be the first -selected frame. Tracker includes these results in copied reports when it -identifies the data as position versus time. +```text +a = 2*A +``` -This calculation uses the position fit. The pixel setting does not assign -errors to Tracker's separate velocity and acceleration data columns. Those -columns combine measurements from several frames, and their errors can be -related because they reuse the same positions. +and its standard error is + +```text +sigma_a = 2*sigma_A. +``` + +For example, + +```text +A = -4.989 +/- 0.023 m/s^2 +``` + +gives + +```text +a = -9.978 +/- 0.046 m/s^2. +``` + +`B` gives the velocity at `t=0`, which need not be the first selected frame. + +Tracker's pixel-uncertainty setting applies to the fitted position measurements. It does not independently assign uncertainties to Tracker's separate velocity and acceleration data columns, because those derived columns reuse measurements from several frames. ## Writing the result in a lab report -Keep full precision during calculations and round when writing the final result. -Tracker displays two significant digits in a positive uncertainty and rounds -the fitted value to the same decimal place. Copied numeric results retain their -full precision. +Keep full precision during calculations and round when reporting the final result. Tracker displays two significant digits in a positive uncertainty and rounds the fitted value to the same decimal place. Copied numeric results retain full precision. + +Record: + +- the selected time interval; +- the fitted equation; +- units; +- the uncertainty choice; +- the fitted result and its uncertainty. + +Also inspect the graph and residuals rather than reporting fit statistics alone. + +`N/A` means that an uncertainty estimate is unavailable. This can occur when parameters are entered manually, a coefficient is held fixed, or the data do not independently determine a coefficient. + +Use **... -> Copy Full Fit Report** when you need the additional statistics. + +## Want to know why? -Record the selected time interval, equation, units, and uncertainty choice. -N/A means that an error estimate is unavailable. This can happen when you enter -parameters manually, hold a coefficient fixed, or provide too little information -to determine a coefficient's uncertainty. Use **... → Copy Full Fit Report** -when you need the additional statistics. +The next step is [Understanding fit uncertainties: the statistics behind the result](fit-uncertainty-details.md). It explains residual variance, degrees of freedom, profile curvature, why other coefficients must be refitted, and the `2.000 +/- 0.063 m/s` example in detail. From a6facb6af3c92675c94666951836907bfbbc012c Mon Sep 17 00:00:00 2001 From: paulnord Date: Mon, 7 Sep 2026 10:53:47 -0500 Subject: [PATCH 18/21] Index fit uncertainty docs by audience --- test/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/README.md b/test/README.md index 04b22b6d..bf5d76f4 100644 --- a/test/README.md +++ b/test/README.md @@ -1,7 +1,14 @@ # Curve-fit regression tests -For interpretation, see the [short uncertainty guide with examples](fit-uncertainty-guide.md). -For the numerical method, see the [Bevington-style implementation note](bevington-fit-uncertainties.md). +## Fit uncertainty documentation + +The fit-uncertainty documentation is arranged in three levels so readers can stop when they have the detail they need: + +1. **Using the result in a lab:** [Understanding fit uncertainties](fit-uncertainty-guide.md) explains what the fitted value and `+/-` uncertainty mean, how to choose the uncertainty model, and how to interpret R-squared and reduced chi-square. +2. **Understanding the statistics:** [The statistics behind the result](fit-uncertainty-details.md) develops residual variance, degrees of freedom, profile curvature, refitting correlated coefficients, rank, and the worked `2.000 +/- 0.063 m/s` example. +3. **Reviewing the implementation:** [Bevington-style fit uncertainties: implementation note](bevington-fit-uncertainties.md) documents the equations as implemented, numerical step rules, code path, edge cases, rank calculation, and regression tests. + +The short guide begins from the principle: **A number from a fit is not the answer; it is a measurement with assumptions attached.** These standalone Java tests require a built OSP or Tracker JAR and a JDK. They use main methods and exit unsuccessfully on assertion failures; no test @@ -38,7 +45,6 @@ unconstrained automatic polynomial fits. Parameter standard errors retain Tracker's profile-curvature method, which may differ from Jacobian covariance estimates for nonlinear models; they are not 95% confidence intervals. - The physics test covers raw/formatted exports with tab/comma delimiters, absent, complete and mixed unit metadata, paste round trips, polynomial parameter units, and the absence of guessed UserFunction units. It also checks residual-estimated From fa99b0de1152b80e62f89fac66fa6e7c7012126e Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 12:12:16 -0400 Subject: [PATCH 19/21] Clarify variance assumptions and connect uncertainty derivation steps --- test/bevington-fit-uncertainties.md | 18 +++++++++++++++-- test/fit-uncertainty-details.md | 30 ++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/test/bevington-fit-uncertainties.md b/test/bevington-fit-uncertainties.md index d6f9abe1..f5220509 100644 --- a/test/bevington-fit-uncertainties.md +++ b/test/bevington-fit-uncertainties.md @@ -303,8 +303,22 @@ the intercept at x=0. They combine to give the intercept variance. If B were held at 1 rather than refitted, changing A would change the prediction by `h*x` instead of `h*(x-x_mean)`. The squared-residual increase would then use `sum(x^2)=0+1+4+9=14` instead of Sxx=5. The resulting slope error would be -`sqrt(0.02/14)=0.0377964473`. It is smaller because B is being treated as known -exactly. That is a different assumption from estimating both A and B. +`sqrt(0.02/14)=0.0377964473`. This comparison keeps the original variance, +0.02, unchanged to isolate the effect of treating B as known exactly. + +Actually fixing B=1 in Tracker's default residual-estimated mode also changes +the variance estimate. The fitted line and SSE remain unchanged in this +example, but the free parameter rank drops from two to one: + +```text +df = 4 - 1 = 3 +sigma_y^2 = 0.04/3 +sigma_A = sqrt((0.04/3)/14) = 0.0308606699924184 +``` + +This second calculation includes both the fixed-intercept assumption and the +re-estimated residual variance. Neither is the original problem of estimating +both A and B. ### Change the supplied measurement uncertainty diff --git a/test/fit-uncertainty-details.md b/test/fit-uncertainty-details.md index 70094549..990dc9fb 100644 --- a/test/fit-uncertainty-details.md +++ b/test/fit-uncertainty-details.md @@ -212,6 +212,13 @@ chi^2_profile(A) = 2 + 5*h^2/0.02. ``` +Compare this with the profile expression from section 2: + +```text +h^2/sigma_A^2 = 5*h^2/0.02 +therefore sigma_A^2 = 0.02/5. +``` + The slope standard error is therefore ```text @@ -248,7 +255,22 @@ sigma_A_fixed_B = sqrt(0.02/14) = 0.0377964... ``` -That smaller number is not a better estimate of the same problem. It answers a different problem in which the intercept is assumed known exactly. +This comparison keeps the original variance, `0.02`, unchanged so that it +isolates the effect of treating the intercept as known exactly. + +If you actually fix `B=1` in Tracker's default residual-estimated mode, Tracker +also recalculates the variance. The line and SSE remain unchanged in this +example, but only one coefficient is now fitted: + +```text +df = 4 - 1 = 3 +sigma_y^2 = 0.04/3 +sigma_A = sqrt((0.04/3)/14) = 0.0308607... +``` + +The two calculations use different uncertainty scales. Neither smaller number +is a better estimate of the original problem, in which both coefficients had +to be determined from the data. ## 5. Supplying a different measurement uncertainty @@ -278,9 +300,11 @@ chi^2 = SSE_min/sigma_y^2 = 1 reduced chi^2 = 1/2 = 0.5. ``` -For `df=2`, the chi-square survival probability is about `Q=0.607`. `Q` is the probability, assuming the model and uncertainty assumptions, of obtaining a chi-square at least this large from repeated data. It is **not** the probability that the model is true. +The usual chi-square probability interpretation assumes independent Gaussian +measurement errors with the stated standard uncertainties and an adequate +model. For `df=2`, the chi-square survival probability is about `Q=0.607`. `Q` is the probability, assuming the model and uncertainty assumptions, of obtaining a chi-square at least this large from repeated data. It is **not** the probability that the model is true. -## 6. When two adjustable coefficients are not two measurements +## 6. When two coefficients cannot be determined separately Consider From 19c55f7af5c56755f9301da8ad71d7f5047a1fe4 Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 12:26:30 -0400 Subject: [PATCH 20/21] Show rounded intercept and uncertainty in the worked example --- test/fit-uncertainty-details.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/fit-uncertainty-details.md b/test/fit-uncertainty-details.md index 990dc9fb..79786e76 100644 --- a/test/fit-uncertainty-details.md +++ b/test/fit-uncertainty-details.md @@ -244,6 +244,12 @@ sigma_B^2 = sigma_y^2 * (1/n + x_mean^2/Sxx) sigma_B = 0.1183216... ``` +so the fitted intercept may be reported approximately as + +```text +B = 1.00 +/- 0.12. +``` + The two coefficient uncertainties differ because slope and intercept affect the fitted line in different ways. ### What if the intercept were known exactly? From 87bd06ce19b46cf948ee254185c9015e13288b7c Mon Sep 17 00:00:00 2001 From: Paul Nord Date: Mon, 7 Sep 2026 12:40:18 -0400 Subject: [PATCH 21/21] Add complete documentation navigation to every fit guide --- test/README.md | 16 +++++++++------- test/bevington-fit-uncertainties.md | 10 ++++++++++ test/fit-report-physics.md | 10 ++++++++++ test/fit-uncertainty-details.md | 10 ++++++++++ test/fit-uncertainty-guide.md | 10 ++++++++++ 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/test/README.md b/test/README.md index bf5d76f4..c2d21d94 100644 --- a/test/README.md +++ b/test/README.md @@ -1,14 +1,16 @@ # Curve-fit regression tests -## Fit uncertainty documentation +## Documentation index -The fit-uncertainty documentation is arranged in three levels so readers can stop when they have the detail they need: +| Document | Contents | +|---|---| +| [Using fit results](fit-uncertainty-guide.md) | Short lab guide to fitted values, uncertainties, and reporting. | +| [Understanding the statistics](fit-uncertainty-details.md) | Statistical reasoning and a worked example. | +| [Bevington implementation](bevington-fit-uncertainties.md) | Profile-curvature method, code, and numerical limitations. | +| [Report definitions and integration](fit-report-physics.md) | Units, uncertainty models, and Tracker integration. | +| [Tests and documentation](README.md) **(this page)** | Regression test instructions and coverage. | -1. **Using the result in a lab:** [Understanding fit uncertainties](fit-uncertainty-guide.md) explains what the fitted value and `+/-` uncertainty mean, how to choose the uncertainty model, and how to interpret R-squared and reduced chi-square. -2. **Understanding the statistics:** [The statistics behind the result](fit-uncertainty-details.md) develops residual variance, degrees of freedom, profile curvature, refitting correlated coefficients, rank, and the worked `2.000 +/- 0.063 m/s` example. -3. **Reviewing the implementation:** [Bevington-style fit uncertainties: implementation note](bevington-fit-uncertainties.md) documents the equations as implemented, numerical step rules, code path, edge cases, rank calculation, and regression tests. - -The short guide begins from the principle: **A number from a fit is not the answer; it is a measurement with assumptions attached.** +## Running the regression tests These standalone Java tests require a built OSP or Tracker JAR and a JDK. They use main methods and exit unsuccessfully on assertion failures; no test diff --git a/test/bevington-fit-uncertainties.md b/test/bevington-fit-uncertainties.md index f5220509..07df5aac 100644 --- a/test/bevington-fit-uncertainties.md +++ b/test/bevington-fit-uncertainties.md @@ -1,5 +1,15 @@ # Bevington-style fit uncertainties: implementation note +## Documentation index + +| Document | Contents | +|---|---| +| [Using fit results](fit-uncertainty-guide.md) | Short lab guide to fitted values, uncertainties, and reporting. | +| [Understanding the statistics](fit-uncertainty-details.md) | Statistical reasoning and a worked example. | +| [Bevington implementation](bevington-fit-uncertainties.md) **(this page)** | Profile-curvature method, code, and numerical limitations. | +| [Report definitions and integration](fit-report-physics.md) | Units, uncertainty models, and Tracker integration. | +| [Tests and documentation](README.md) | Regression test instructions and coverage. | + Tracker first fits the coefficients of a function to the measured data. It then estimates an error for each free coefficient by changing that coefficient, refitting the others, and measuring the increase in chi-square. This second diff --git a/test/fit-report-physics.md b/test/fit-report-physics.md index d908cca1..af48af6d 100644 --- a/test/fit-report-physics.md +++ b/test/fit-report-physics.md @@ -1,5 +1,15 @@ # Fit report units and uncertainty models +## Documentation index + +| Document | Contents | +|---|---| +| [Using fit results](fit-uncertainty-guide.md) | Short lab guide to fitted values, uncertainties, and reporting. | +| [Understanding the statistics](fit-uncertainty-details.md) | Statistical reasoning and a worked example. | +| [Bevington implementation](bevington-fit-uncertainties.md) | Profile-curvature method, code, and numerical limitations. | +| [Report definitions and integration](fit-report-physics.md) **(this page)** | Units, uncertainty models, and Tracker integration. | +| [Tests and documentation](README.md) | Regression test instructions and coverage. | + The coefficient optimizer is unchanged. A positive constant weight multiplies the least-squares objective by a constant and therefore has the same minimizer; changing the uncertainty controls recomputes profile errors on a cloned function diff --git a/test/fit-uncertainty-details.md b/test/fit-uncertainty-details.md index 79786e76..9cd77675 100644 --- a/test/fit-uncertainty-details.md +++ b/test/fit-uncertainty-details.md @@ -1,5 +1,15 @@ # Understanding fit uncertainties: the statistics behind the result +## Documentation index + +| Document | Contents | +|---|---| +| [Using fit results](fit-uncertainty-guide.md) | Short lab guide to fitted values, uncertainties, and reporting. | +| [Understanding the statistics](fit-uncertainty-details.md) **(this page)** | Statistical reasoning and a worked example. | +| [Bevington implementation](bevington-fit-uncertainties.md) | Profile-curvature method, code, and numerical limitations. | +| [Report definitions and integration](fit-report-physics.md) | Units, uncertainty models, and Tracker integration. | +| [Tests and documentation](README.md) | Regression test instructions and coverage. | + > **A number from a fit is not the answer; it is a measurement with assumptions attached.** This is the middle layer of the Tracker fit-uncertainty documentation. diff --git a/test/fit-uncertainty-guide.md b/test/fit-uncertainty-guide.md index 7cc24d64..cf278509 100644 --- a/test/fit-uncertainty-guide.md +++ b/test/fit-uncertainty-guide.md @@ -1,5 +1,15 @@ # Understanding fit uncertainties +## Documentation index + +| Document | Contents | +|---|---| +| [Using fit results](fit-uncertainty-guide.md) **(this page)** | Short lab guide to fitted values, uncertainties, and reporting. | +| [Understanding the statistics](fit-uncertainty-details.md) | Statistical reasoning and a worked example. | +| [Bevington implementation](bevington-fit-uncertainties.md) | Profile-curvature method, code, and numerical limitations. | +| [Report definitions and integration](fit-report-physics.md) | Units, uncertainty models, and Tracker integration. | +| [Tests and documentation](README.md) | Regression test instructions and coverage. | + > **A number from a fit is not the answer; it is a measurement with assumptions attached.** Suppose Tracker reports a velocity of **2.000 +/- 0.063 m/s**.