Проверяйте градиенты, прежде чем доверять обратному распространению

pub fn quadratic(point: f64) -> f64 {
    point * point
}

/// Checks either the expected derivative `6` or a deliberately wrong candidate.
pub fn quadratic_gradient_check(analytic: f64) -> Result<ScalarGradientCheck, GradCheckError> {
    scalar_gradient_check(3.0, analytic, 0.1, STEP_TOLERANCE, quadratic)
}

/// Approximates one derivative from the actual representable probe spacings.
///
/// The callback is evaluated exactly three times, in minus, center, plus order.
/// A meaningful gradient check requires a deterministic, side-effect-free
/// objective that is differentiable and sufficiently smooth throughout the
/// closed probe interval. Known or possible kinks must be excluded or examined
/// with [`compare_one_sided_slopes`]; a passing comparison is evidence for the
/// sampled smooth point, not a universal proof of a derivative.
pub fn central_difference(
    point: f64,
    step: f64,
    mut evaluate: impl FnMut(f64) -> f64,
) -> Result<CentralDifference, GradCheckError> {
    let geometry = perturbations(point, step)?;

    let minus_value = evaluate(geometry.minus_point);
    if !minus_value.is_finite() {
        return Err(GradCheckError::NonFiniteEvaluation {
            side: DifferenceSide::Minus,
            value: minus_value,
        });
    }

    let center_value = evaluate(point);
    if !center_value.is_finite() {
        return Err(GradCheckError::NonFiniteEvaluation {
            side: DifferenceSide::Center,
            value: center_value,
        });
    }

    let plus_value = evaluate(geometry.plus_point);
    if !plus_value.is_finite() {
        return Err(GradCheckError::NonFiniteEvaluation {
            side: DifferenceSide::Plus,
            value: plus_value,
        });
    }

    let left_slope = (center_value - minus_value) / geometry.minus_spacing;
    if !left_slope.is_finite() {
        return Err(GradCheckError::NonFiniteOneSidedSlope {
            side: DifferenceSide::Minus,
            value: left_slope,
        });
    }
    let right_slope = (plus_value - center_value) / geometry.plus_spacing;
    if !right_slope.is_finite() {
        return Err(GradCheckError::NonFiniteOneSidedSlope {
            side: DifferenceSide::Plus,
            value: right_slope,
        });
    }

    let derivative = geometry.left_weight * left_slope + geometry.right_weight * right_slope;
    if !derivative.is_finite() {
        return Err(GradCheckError::NonFiniteNumericalGradient { value: derivative });
    }

    Ok(CentralDifference {
        point,
        requested_step: step,
        minus_point: geometry.minus_point,
        plus_point: geometry.plus_point,
        minus_spacing: geometry.minus_spacing,
        plus_spacing: geometry.plus_spacing,
        minus_value,
        center_value,
        plus_value,
        left_slope,
        right_slope,
        left_weight: geometry.left_weight,
        right_weight: geometry.right_weight,
        stencil: geometry.stencil,
        derivative,
    })
}

/// Compares gradients after scaling both by the larger magnitude or one.
pub fn compare_gradients(
    analytic: f64,
    numerical: f64,
    tolerance: f64,
) -> Result<GradientComparison, GradCheckError> {
    validate_tolerance(tolerance)?;
    if !analytic.is_finite() {
        return Err(GradCheckError::NonFiniteAnalyticGradient { value: analytic });
    }
    if !numerical.is_finite() {
        return Err(GradCheckError::NonFiniteNumericalGradient { value: numerical });
    }

    let scale = 1.0_f64.max(analytic.abs()).max(numerical.abs());
    let absolute_error = (analytic - numerical).abs();
    let scaled_error = (analytic / scale - numerical / scale).abs();

    Ok(GradientComparison {
        analytic,
        numerical,
        absolute_error,
        scale,
        scaled_error,
        tolerance,
        passed: scaled_error <= tolerance,
    })
}

/// Compares the recorded left and right secant slopes without claiming proof.
///
/// A failed diagnostic can indicate a nondifferentiable kink, a step that is
/// too large for the local curvature, or floating-point rounding. A passing
/// diagnostic cannot establish differentiability on its own.
pub fn compare_one_sided_slopes(
    difference: &CentralDifference,
    tolerance: f64,
) -> Result<OneSidedSlopeComparison, GradCheckError> {
    validate_tolerance(tolerance)?;
    let left = difference.left_slope;
    if !left.is_finite() {
        return Err(GradCheckError::NonFiniteOneSidedSlope {
            side: DifferenceSide::Minus,
            value: left,
        });
    }
    let right = difference.right_slope;
    if !right.is_finite() {
        return Err(GradCheckError::NonFiniteOneSidedSlope {
            side: DifferenceSide::Plus,
            value: right,
        });
    }
    let scale = 1.0_f64.max(left.abs()).max(right.abs());
    let absolute_gap = (left - right).abs();
    let scaled_gap = (left / scale - right / scale).abs();
    Ok(OneSidedSlopeComparison {
        left,
        right,
        absolute_gap,
        scale,
        scaled_gap,
        tolerance,
        consistent: scaled_gap <= tolerance,
    })
}

/// Runs a central difference and compares it with one analytic candidate.
///
/// The objective has the same smoothness, determinism, and side-effect
/// requirements as [`central_difference`].
pub fn scalar_gradient_check(
    point: f64,
    analytic: f64,
    step: f64,
    tolerance: f64,
    evaluate: impl FnMut(f64) -> f64,
) -> Result<ScalarGradientCheck, GradCheckError> {
    validate_tolerance(tolerance)?;
    let difference = central_difference(point, step, evaluate)?;
    let comparison = compare_gradients(analytic, difference.derivative, tolerance)?;
    Ok(ScalarGradientCheck {
        difference,
        comparison,
    })
}

/// A rejected numerical derivative, comparison, sample request, or tensor read.
#[derive(Clone, Debug, PartialEq)]
pub enum GradCheckError {
    InvalidStep {
        step: f64,
    },
    NonFinitePoint {
        point: f64,
    },
    PerturbationNotFinite {
        side: DifferenceSide,
        point: f64,
        step: f64,
    },
    PerturbationUnchanged {
        side: DifferenceSide,
        point: f64,
        step: f64,
    },
    InvalidActualSpacing {
        side: DifferenceSide,
        point: f64,
        probe: f64,
        spacing: f64,
    },
    NonFiniteStencilWeight {
        side: DifferenceSide,
        value: f64,
    },
    NonFiniteEvaluation {
        side: DifferenceSide,
        value: f64,
    },
    NonFiniteOneSidedSlope {
        side: DifferenceSide,
        value: f64,
    },
    NonFiniteNumericalGradient {
        value: f64,
    },
    InvalidTolerance {
        tolerance: f64,
    },
    NonFiniteAnalyticGradient {
        value: f64,
    },
    GradientShapeMismatch {
        parameters: Vec<usize>,
        analytic: Vec<usize>,
    },
    ShapeOverflow,
    EmptyTensor,
    ZeroSamples,
    SampleAllocationFailed {
        samples: usize,
        rank: usize,
    },
    View(TensorViewError),
    AtCoordinate {
        coordinate: Vec<usize>,
        source: Box<GradCheckError>,
    },
}

impl fmt::Display for GradCheckError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::InvalidStep { step } => write!(
                formatter,
                "central-difference requested step {step:?} must be positive and finite"
            ),
            Self::NonFinitePoint { point } => {
                write!(formatter, "gradient-check point {point:?} must be finite")
            }
            Self::PerturbationNotFinite { side, point, step } => write!(
                formatter,
                "{side} perturbation from point {point:?} by step {step:?} is not finite"
            ),
            Self::PerturbationUnchanged { side, point, step } => write!(
                formatter,
                "{side} perturbation from point {point:?} by step {step:?} rounds back to the point"
            ),
            Self::InvalidActualSpacing {
                side,
                point,
                probe,
                spacing,
            } => write!(
                formatter,
                "{side} actual spacing from point {point:?} to probe {probe:?} must be positive and finite, got {spacing:?}"
            ),
            Self::NonFiniteStencilWeight { side, value } => {
                write!(formatter, "{side} stencil weight is not finite: {value:?}")
            }
            Self::NonFiniteEvaluation { side, value } => {
                write!(formatter, "{side} function evaluation returned {value:?}")
            }
            Self::NonFiniteOneSidedSlope { side, value } => {
                write!(formatter, "{side} one-sided slope is not finite: {value:?}")
            }
            Self::NonFiniteNumericalGradient { value } => {
                write!(
                    formatter,
                    "central difference produced non-finite gradient {value:?}"
                )
            }
            Self::InvalidTolerance { tolerance } => write!(
                formatter,
                "gradient-check tolerance {tolerance:?} must be finite and nonnegative"
            ),
            Self::NonFiniteAnalyticGradient { value } => {
                write!(formatter, "analytic gradient {value:?} must be finite")
            }
            Self::GradientShapeMismatch {
                parameters,
                analytic,
            } => write!(
                formatter,
                "parameter shape {parameters:?} does not match analytic-gradient shape {analytic:?}"
            ),
            Self::ShapeOverflow => formatter.write_str("sampled tensor shape overflows usize"),
            Self::EmptyTensor => {
                formatter.write_str("sampled tensor gradient check needs at least one value")
            }
            Self::ZeroSamples => {
                formatter.write_str("sampled tensor gradient check needs at least one sample")
            }
            Self::SampleAllocationFailed { samples, rank } => write!(
                formatter,
                "cannot allocate {samples} sampled coordinates of rank {rank}"
            ),
            Self::View(error) => error.fmt(formatter),
            Self::AtCoordinate { coordinate, source } => {
                write!(
                    formatter,
                    "gradient check at coordinate {coordinate:?} failed: {source}"
                )
            }
        }
    }
}

impl Error for GradCheckError {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        match self {
            Self::View(error) => Some(error),
            Self::AtCoordinate { source, .. } => Some(source.as_ref()),
            _ => None,
        }
    }
}

impl From<TensorViewError> for GradCheckError {
    fn from(error: TensorViewError) -> Self {
        Self::View(error)
    }
}

/// Selects unique, ordered coordinates without randomness or hidden state.
pub fn sample_tensor_coordinates(
    shape: &[usize],
    max_samples: usize,
) -> Result<Vec<SampledCoordinate>, GradCheckError> {
    if max_samples == 0 {
        return Err(GradCheckError::ZeroSamples);
    }
    let elements = element_count(shape)?;
    if elements == 0 {
        return Err(GradCheckError::EmptyTensor);
    }
    let samples = max_samples.min(elements);
    let mut coordinates = Vec::new();
    coordinates
        .try_reserve_exact(samples)
        .map_err(|_| GradCheckError::SampleAllocationFailed {
            samples,
            rank: shape.len(),
        })?;

    for sample in 0..samples {
        let flat_index = if samples == 1 {
            elements / 2
        } else {
            let numerator = (sample as u128) * ((elements - 1) as u128);
            (numerator / ((samples - 1) as u128)) as usize
        };
        coordinates.push(SampledCoordinate {
            flat_index,
            coordinate: coordinate_from_offset(shape, flat_index)?,
        });
    }
    Ok(coordinates)
}

/// Checks deterministic tensor coordinates while restoring every probed value.
///
/// The parameter tensor is restored on every ordinary `Ok` or `Err` path. A
/// panic inside `objective` is deliberately outside this reference guarantee.
/// The objective must be deterministic, side-effect-free, and differentiable
/// with sufficient smoothness across every sampled probe interval; known or
/// possible kinks are outside this comparison's contract.
pub fn sampled_tensor_gradient_check(
    parameters: &mut Tensor,
    analytic: &TensorView<'_>,
    step: f64,
    tolerance: f64,
    max_samples: usize,
    mut objective: impl FnMut(&Tensor) -> f64,
) -> Result<TensorGradientCheck, GradCheckError> {
    validate_step(step)?;
    validate_tolerance(tolerance)?;
    if parameters.shape() != analytic.shape() {
        return Err(GradCheckError::GradientShapeMismatch {
            parameters: parameters.shape().to_vec(),
            analytic: analytic.shape().to_vec(),
        });
    }

    let samples = sample_tensor_coordinates(parameters.shape(), max_samples)?;
    let mut candidates = Vec::new();
    candidates.try_reserve_exact(samples.len()).map_err(|_| {
        GradCheckError::SampleAllocationFailed {
            samples: samples.len(),
            rank: parameters.rank(),
        }
    })?;

    // Validate every selected coordinate before the first objective call.
    for sample in &samples {
        let point = parameters.as_slice()[sample.flat_index];
        perturbations(point, step).map_err(|error| at_coordinate(&sample.coordinate, error))?;
        let analytic_value = *analytic
            .get(&sample.coordinate)
            .map_err(GradCheckError::View)?;
        if !analytic_value.is_finite() {
            return Err(at_coordinate(
                &sample.coordinate,
                GradCheckError::NonFiniteAnalyticGradient {
                    value: analytic_value,
                },
            ));
        }
        candidates.push((sample.clone(), point, analytic_value));
    }

    let mut checks = Vec::new();
    checks.try_reserve_exact(candidates.len()).map_err(|_| {
        GradCheckError::SampleAllocationFailed {
            samples: candidates.len(),
            rank: parameters.rank(),
        }
    })?;

    for (sample, point, analytic_value) in candidates {
        let difference = central_difference(point, step, |probe| {
            parameters.as_mut_slice()[sample.flat_index] = probe;
            let value = objective(parameters);
            parameters.as_mut_slice()[sample.flat_index] = point;
            value
        })
        .map_err(|error| at_coordinate(&sample.coordinate, error))?;
        let comparison = compare_gradients(analytic_value, difference.derivative, tolerance)
            .map_err(|error| at_coordinate(&sample.coordinate, error))?;
        checks.push(CoordinateGradientCheck {
            flat_index: sample.flat_index,
            coordinate: sample.coordinate,
            difference,
            comparison,
        });
    }

    Ok(TensorGradientCheck {
        shape: parameters.shape().to_vec(),
        requested_samples: max_samples,
        passed: checks.iter().all(|check| check.comparison.passed),
        checks,
    })
}

/// Applies `(normalized probabilities - one_hot(target)) / batch_size` through a separate local path.
///
/// This frozen two-row analytic path deliberately implements its own row traversal,
/// maximum shift, exponential sum, normalization, target subtraction, and
/// batch scaling without calling the production probability or indexed-NLL
/// helpers. Both paths still share the input values and targets, IEEE `f64`
/// arithmetic and its primitive exponential, `Tensor` storage, and row-major
/// index conventions.
pub fn hand_derived_nll_gradient(logits: &Tensor) -> Result<Tensor, Box<dyn Error>> {
    if logits.shape() != LOGIT_SHAPE {
        return Err(format!(
            "the local Chapter 13 analytic path requires shape {LOGIT_SHAPE:?}, got {:?}",
            logits.shape()
        )
        .into());
    }

    let columns = LOGIT_SHAPE[1];
    let mut values = vec![0.0; LOGIT_VALUES.len()];
    for (row, &target) in TARGETS.iter().enumerate() {
        let start = row * columns;
        let row_logits = &logits.as_slice()[start..start + columns];
        let maximum = row_logits.iter().copied().fold(f64::NEG_INFINITY, f64::max);
        if !maximum.is_finite() {
            return Err(format!("oracle row {row} has no finite maximum").into());
        }

        let mut normalizer = 0.0;
        for (column, &logit) in row_logits.iter().enumerate() {
            if !logit.is_finite() {
                return Err(format!("oracle logit at [{row}, {column}] is not finite").into());
            }
            let weight = (logit - maximum).exp();
            values[start + column] = weight;
            normalizer += weight;
        }
        if !normalizer.is_finite() || normalizer <= 0.0 {
            return Err(
                format!("oracle row {row} has invalid normalization {normalizer:?}").into(),
            );
        }

        for column in 0..columns {
            values[start + column] /= normalizer;
        }
        values[start + target] -= 1.0;
    }
    for gradient in &mut values {
        *gradient /= TARGETS.len() as f64;
    }
    Ok(Tensor::from_vec(LOGIT_SHAPE.to_vec(), values)?)
}

/// Checks four deterministic vocabulary-logit coordinates against indexed NLL.
pub fn tiny_nll_gradient_example() -> Result<TinyNllGradientExample, Box<dyn Error>> {
    let mut logits = logits()?;
    let analytic = hand_derived_nll_gradient(&logits)?;
    let loss = indexed_mean_nll(&logits.view(), 1, &TARGETS)?;
    let original_bits = logits
        .as_slice()
        .iter()
        .map(|value| value.to_bits())
        .collect::<Vec<_>>();
    let check = sampled_tensor_gradient_check(
        &mut logits,
        &analytic.view(),
        TENSOR_STEP,
        TENSOR_TOLERANCE,
        TENSOR_SAMPLES,
        |candidate| {
            indexed_mean_nll(&candidate.view(), 1, &TARGETS)
                .expect("the frozen finite logits and targets remain valid")
        },
    )?;
    let restored_exactly = logits
        .as_slice()
        .iter()
        .map(|value| value.to_bits())
        .eq(original_bits);

    Ok(TinyNllGradientExample {
        logits,
        analytic,
        loss,
        check,
        restored_exactly,
    })
}

    let correct = quadratic_gradient_check(6.0)?;
    let wrong = quadratic_gradient_check(5.5)?;
    let rounded = rounded_identity_gradient_check()?;
    let kink = absolute_kink_diagnostic()?;
    let scan = cubic_step_scan()?;
    let nll = tiny_nll_gradient_example()?;
    let collapsed = central_difference(1.0, 1.0e-20, |point| point * point).unwrap_err();

/// Runs the same central difference from truncation-dominated to rounded probes.
pub fn cubic_step_scan() -> Result<Vec<StepScanRecord>, GradCheckError> {
    STEP_SCAN
        .iter()
        .zip(STEP_PHASES)
        .map(|(&step, phase)| {
            scalar_gradient_check(CUBIC_POINT, CUBIC_ANALYTIC, step, STEP_TOLERANCE, cubic)
                .map(|check| StepScanRecord { phase, check })
        })
        .collect()
}